diff --git a/Cargo.lock b/Cargo.lock index 68672ad8580..cf058c22dd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3681,6 +3681,7 @@ dependencies = [ "scrypt", "serde_json", "sha2 0.10.9", + "toyos", "x25519-dalek", ] diff --git a/issues/intels-store-forwarding-predictor-times-constant-time-code-by-its-data.md b/issues/intels-store-forwarding-predictor-times-constant-time-code-by-its-data.md new file mode 100644 index 00000000000..e4a80f76c2f --- /dev/null +++ b/issues/intels-store-forwarding-predictor-times-constant-time-code-by-its-data.md @@ -0,0 +1,40 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# Intel's store-forwarding predictor times constant-time code by its data + +On the T14 (Tiger Lake, signature 0x806c1, microcode 0xbe), toyos-crypto's +P-256 and P-384 public key and agreement at 9217793d2, whose branches and +addresses do not depend on the scalar, leaked through timing: max |t| 69 to +81 over 1,000,000 fixed-versus-random measurements, the fixed scalar 1 about +90 ticks slower in 316,000 for P-256. Turning on PSFD alone (`IA32_SPEC_CTRL` +bit 7), which disables only the Fast Store Forwarding Predictor, removed it, +and so did DOITM and SSBD. The T14 enumerates no `DDP_CTRL`, so it has no +data-dependent prefetcher. + +- Evidence: the off/on pair, + https://github.com/ToyOSOrg/evidence/tree/7fb806b73ce0397b9cbec9e1da1df910ba8f4785/pr-1038/r1/probe, + and the attribution boot by mode, + https://github.com/ToyOSOrg/evidence/tree/56a9678bcec3939074783b0f0a363eb14c1a25da/combined/fsfp-attribution. +- Intel lists FSFP on Ice Lake U/Y and SP, Lakefield, Tiger Lake U/H, Rocket + Lake and Alder Lake P/M/S, and says later parts declare it by enumerating + PSFD (CPUID.(7,2).EDX[0]), as the T14 does too (EDX 0x1). It does not + document what makes the predictor forward wrongly, so no coding rule + avoids it. +- Current code passes on the T14 with every such control off. That rests on + avoiding the value pattern that triggered it (#1013 keeps the running point + off the identity), which a test of chosen secrets cannot extend to every + secret. +- By the owner's ruling (2026-10-26) ToyOS switches no DOITM on x86-64: + constant time there rests on the code, as it must on AMD, which has no + such mode. `toyos::timing` reports x86-64 unprotected. + +**Owner**: the crypto track. **Exit**: a program holding secrets on a CPU +with FSFP runs with it disabled, by PSFD or SSBD through the per-program flag +of `issues/no-program-runs-with-speculative-store-bypass-disabled.md`, or +Intel documents which data the predictor depends on and toyos-crypto is shown +to keep secrets out of it. **Oracle**: `crypto_secret_timing_curves` on the +T14 at a revision that leaked, with the control on and off. diff --git a/issues/the-avx2-poly1305-runs-under-intels-mxcsr-dependent-timing.md b/issues/the-avx2-poly1305-runs-under-intels-mxcsr-dependent-timing.md new file mode 100644 index 00000000000..c0a109c7535 --- /dev/null +++ b/issues/the-avx2-poly1305-runs-under-intels-mxcsr-dependent-timing.md @@ -0,0 +1,26 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# The AVX2 Poly1305 runs under Intel's MXCSR-dependent timing + +Intel's "MXCSR Configuration Dependent Timing" guidance says that on a CPU +that does not enumerate `MCDT_NO` (CPUID.(07H,2):EDX[5]), some vector +instructions can retire up to one cycle later on particular data under an +MXCSR other than 0x1FBF. `VPMULUDQ` is on its list, and toyos-crypto's AVX2 +Poly1305 kernel (`lib/crypto/src/arch/x86_64/poly1305.rs`) runs it on the +secret key's limbs under the default MXCSR (0x1F80). The T14 does not +enumerate `MCDT_NO`: its CPUID.(07H,2):EDX reads 0x1, in the boot at +https://github.com/ToyOSOrg/evidence/tree/56a9678bcec3939074783b0f0a363eb14c1a25da/combined/fsfp-attribution. +DOITM does not cover it, and by the owner's ruling (2026-10-26) ToyOS +switches no DOITM on x86-64 anyway. + +Whether the delay is measurable on our benches is unmeasured. + +**Owner**: the crypto track. **Exit**: where `MCDT_NO` is clear, the AVX2 +Poly1305 kernel runs under Intel's sequence (`STMXCSR`; `LDMXCSR` 0x1FBF; +`LFENCE` before the kernel; `LFENCE`; `LDMXCSR` of the saved value after), +its cost measured on the T14. **Oracle**: Intel's instruction list and the +CPU's `MCDT_NO` bit. diff --git a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md index 5e681b34e21..12927162fd2 100644 --- a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md +++ b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md @@ -54,6 +54,14 @@ review has been done, and passing vectors does not show it is safer than ring. 2.6 GB/s against the portable rounds' 0.52, and SHA-512 at 1.36 against 0.83. ToyOS's own AArch64 targets do not promise them and nothing detects them at run time, so they hash portably; see the weaknesses. +- **On ToyOS every operation that takes a secret runs under the thread's + timing protection** (`src/timing.rs`, `toyos::timing`): `PSTATE.DIT` on + AArch64. That covers ciphers, MACs, KDFs, keyed SipHash, `ct::equal` and + the key operations. A plain hash takes none, so a caller hashing a secret + takes `toyos_crypto::timing::protect` itself, as `toyos-ssh`'s exchange + hash and key derivation do. x86-64 has no such mode a thread sets, and + ToyOS turns Intel's DOITM on for none, so constant time there rests on the + code, as on AMD. ## Consumers and what each needs @@ -499,6 +507,28 @@ Not consumers: 2.65; null 1.82 and 1.04, leaky 210.6 and 194.7. Its Ed25519 run is pending. Evidence: https://github.com/ToyOSOrg/evidence/tree/6488b4fbb6785e65332d03fb504fe3849d89810a/pr-1046/r4 + - **#1063's P-256 register kernels at 9a5d4fad5** (with the AVX2 + table read), 1,000,000 measurements a routine, no CPU timing mode: + - The T14 (0x806c1, microcode 0xbe): P-256 public key max |t| + 3.20 and agreement 1.51; P-384 public key 1.19 and agreement + 1.42; Ed25519 2.73. Null 1.22 and 1.70, leaky 855.8 and 890.6. + - The AMD bench (0x860f01, 0x860010d): P-256 2.99 and 1.49; P-384 + 2.47 and 2.43; Ed25519 1.66. Null 1.14 and 2.12, leaky 140.1 + and 169.3. + - Evidence: + https://github.com/ToyOSOrg/evidence/tree/aa3369406af00b46a4b76476ef4cea7df040e30f/combined/r3 + - Its speed A/B against main, three base/head pairs a bench: P-256 + ECDH 0.75–0.82 of main and verify 0.76–0.85, but for one T14 + boot in which ring ran 1.10x slower too. + - **A measured layout effect:** AMD asserted `chacha20-poly1305/key` + rose from 5.3 ns to 6.0, 6.9 and 6.0 ns in the three pairs, with ring + unchanged. Its instructions are the same in both builds; only + `SealingKey::chacha20_poly1305` moved, from a 64-byte boundary to + 32 bytes past one. The T14 and the unasserted build show + 1.00–1.04. Evidence: + https://github.com/ToyOSOrg/evidence/tree/58dcc2314053075b410ed6ccdd48f0e2b68c9c1f/combined/r4 + and + https://github.com/ToyOSOrg/evidence/tree/d6e8e35334d4f83b5140e7dedd8b4d9f1f49b338/combined/r5 - Their judge asks only that each job ran and its controls held; the verdicts are this record's. The first boots, of the fast program at ec5a0cd18, read LEAKS everywhere: that harness made each random @@ -652,6 +682,14 @@ Not consumers: ## Weaknesses until the exits hold +- On x86-64 nothing but the code keeps secret work's timing independent of + its data. The T14's store-forwarding predictor once timed P-256 by its + data, which DOITM, PSFD or SSBD removed + (`issues/intels-store-forwarding-predictor-times-constant-time-code-by-its-data.md`). + Intel's MXCSR-dependent timing can delay the AVX2 Poly1305's `VPMULUDQ` + on its data + (`issues/the-avx2-poly1305-runs-under-intels-mxcsr-dependent-timing.md`). + **Owner**: this track. **Exit**: both issues'. - Erasure is what `lib/crypto/src/secret.rs`'s contract states: secret owners, each key agreement's shared value and named workspaces are erased. Since the parity work's 5x51 field (#992), 2^255 - 19 field diff --git a/kernel/src/arch/aarch64/control_regs.rs b/kernel/src/arch/aarch64/control_regs.rs index 8c6b24b9494..d342369c6c8 100644 --- a/kernel/src/arch/aarch64/control_regs.rs +++ b/kernel/src/arch/aarch64/control_regs.rs @@ -9,7 +9,7 @@ //! and the GIC architecture specification (IHI 0069H), chapter 12, for the //! `ICC_` registers. -use core::sync::atomic::AtomicU64; +use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; use crate::log; @@ -94,6 +94,33 @@ macro_rules! read { }}; } +/// FEAT_DIT, `ID_AA64PFR0_EL1.DIT` (bits 51:48) nonzero, as the boot CPU +/// answered it: every other CPU must answer the same, since a thread that sets +/// `PSTATE.DIT` on one runs on any. Zero until the boot CPU's [`check`]. +static DIT: AtomicU8 = AtomicU8::new(0); +const DIT_ABSENT: u8 = 1; +const DIT_PRESENT: u8 = 2; + +/// Whether a user thread can set `PSTATE.DIT` for itself: every CPU has +/// FEAT_DIT. EL0 writes it, every exception from EL0 saves it in `SPSR_EL1` +/// in the thread's `trap::Frame` and the return puts it back, and a thread +/// starts with `SPSR_EL1` zero, so it is each thread's own with no switch work. +pub fn user_timing_control() -> bool { + DIT.load(Ordering::Acquire) == DIT_PRESENT +} + +/// This CPU's FEAT_DIT, held to the boot CPU's. +fn hold_dit() { + let mine = read!("id_aa64pfr0_el1") >> 48 & 0xF != 0; + let code = if mine { DIT_PRESENT } else { DIT_ABSENT }; + match DIT.compare_exchange(0, code, Ordering::Release, Ordering::Acquire) { + Ok(_) => log!("control registers: the boot CPU {} FEAT_DIT, as every other must", if mine { "has" } else { "has no" }), + Err(machine) => { + assert!(machine == code, "control registers: this CPU's FEAT_DIT is {mine} and the boot CPU's is {}", !mine) + } + } +} + /// `TCR_EL1` whole, as this CPU's physical address range makes it. pub fn tcr() -> u64 { TCR | (read!("id_aa64mmfr0_el1") & 0xF) << TCR_IPS_SHIFT @@ -116,6 +143,7 @@ pub fn check(el: u64) { for (name, live, value) in declared { assert_eq!(live, value, "control registers: {name} holds {live:#x}, and the declaration says {value:#x}"); } + hold_dit(); // What the drop from EL2 left, or the EL1 entry kept: EL1, on `SP_EL1`. let (now, spsel) = (read!("CurrentEL") >> 2 & 3, read!("SPSel") & 1); assert_eq!((now, spsel), (1, 1), "control registers: running at EL{now} on SP_EL{spsel}, not EL1 on SP_EL1"); diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 436ed9dcffb..aa82a921422 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -399,6 +399,15 @@ pub fn pcid_active() -> bool { DECLARED_CR4.load(Ordering::Acquire) & cr4::PCIDE != 0 } +/// Whether a user thread can set data-independent timing for itself: never +/// on x86-64. Intel's DOITM (`IA32_UARCH_MISC_CTL[0]`) is Ring 0's, and this +/// kernel turns it on for no thread +/// (`issues/intels-store-forwarding-predictor-times-constant-time-code-by-its-data.md`); +/// AMD has no such mode. +pub fn user_timing_control() -> bool { + false +} + /// Drops every translation this CPU holds, global ones included (SDM Vol. 3A /// §4.10.4.1, APM Vol. 2 §5.5): `PGE` cleared and put back where it is set; /// where it is not, nothing is global and a `CR3` reload drops the rest. For a diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 575f26ee549..03287b55cb9 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -358,6 +358,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } } SYS_CPU_COUNT => crate::smp::cpu_count() as u64, + SYS_TIMING_CONTROL => u64::from(crate::arch::control_regs::user_timing_control()), SYS_FUTEX_WAIT => match UserAddr::checked(a1) { Some(addr) => process::futex_wait(addr, a2 as u32, a3), None => bad_addr, diff --git a/lib/crypto/Cargo.toml b/lib/crypto/Cargo.toml index 775d4f313ae..9d19c46b241 100644 --- a/lib/crypto/Cargo.toml +++ b/lib/crypto/Cargo.toml @@ -21,6 +21,10 @@ rustls-provider = ["dep:rustls"] [dependencies] rustls = { version = "0.23.45", optional = true, default-features = false, features = ["custom-provider", "tls12"] } +# The thread's timing protection around secret work (`src/timing.rs`). +[target.'cfg(target_os = "toyos")'.dependencies] +toyos = { path = "../../sdk/toyos" } + # The oracles every primitive is held against, on the host alone. [dev-dependencies] ring = "0.17.14" @@ -44,3 +48,4 @@ num-bigint = "0.4" [lints.rust] unsafe_op_in_unsafe_fn = "deny" +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_os, values("toyos"))'] } diff --git a/lib/crypto/src/aead.rs b/lib/crypto/src/aead.rs index 07d634d4453..4b415ccf4a1 100644 --- a/lib/crypto/src/aead.rs +++ b/lib/crypto/src/aead.rs @@ -56,9 +56,11 @@ impl SealingKey { } #[cfg(feature = "rustls-provider")] pub(crate) fn initialize(&mut self, key: &[u8]) { + let _protection = crate::timing::protect(); self.key.initialize(key); } pub fn aes128_gcm(key: [u8; 16], iv: [u8; 12]) -> Self { + let _protection = crate::timing::protect(); let key = Secret(key); Self { key: Key::Gcm(crate::gcm::Gcm::aes128(&key.0)), @@ -68,6 +70,7 @@ impl SealingKey { } } pub fn aes256_gcm(key: [u8; 32], iv: [u8; 12]) -> Self { + let _protection = crate::timing::protect(); let key = Secret(key); Self { key: Key::Gcm(crate::gcm::Gcm::aes256(&key.0)), @@ -90,6 +93,7 @@ impl SealingKey { aad: &[u8], plaintext: &mut [u8], ) -> Result<([u8; 12], [u8; 16]), Error> { + let _protection = crate::timing::protect(); if self.exhausted { return Err(Error::CounterExhausted); } @@ -122,15 +126,18 @@ impl OpeningKey { } #[cfg(feature = "rustls-provider")] pub(crate) fn initialize(&mut self, key: &[u8]) { + let _protection = crate::timing::protect(); self.key.initialize(key); } pub fn aes128_gcm(key: [u8; 16]) -> Self { + let _protection = crate::timing::protect(); let key = Secret(key); Self { key: Key::Gcm(crate::gcm::Gcm::aes128(&key.0)), } } pub fn aes256_gcm(key: [u8; 32]) -> Self { + let _protection = crate::timing::protect(); let key = Secret(key); Self { key: Key::Gcm(crate::gcm::Gcm::aes256(&key.0)), @@ -148,6 +155,7 @@ impl OpeningKey { ciphertext: &mut [u8], tag: &[u8], ) -> Result<(), Error> { + let _protection = crate::timing::protect(); if tag.len() != 16 { return Err(Error::Length); } diff --git a/lib/crypto/src/arch/mod.rs b/lib/crypto/src/arch/mod.rs index 92be17e1f93..4d9a9c2f600 100644 --- a/lib/crypto/src/arch/mod.rs +++ b/lib/crypto/src/arch/mod.rs @@ -68,23 +68,12 @@ pub(crate) fn rsa_mul(a: &[u64], b: &[u64], m: &[u64], n0: u64, n: usize, out: & }) } -/// a b / 2^256 mod P-256's p, for a, b < p, by an instruction path; `None` -/// where the CPU has no such path. The time does not depend on a or b. -#[inline] -pub(crate) fn p256_mul(a: &[u64; 4], b: &[u64; 4]) -> Option<[u64; 4]> { - hosted!(x86_64::p256_mul(a, b), { - let _ = (a, b); - None - }) -} - -/// a^2 / 2^256 mod P-256's p, for a < p, as [`p256_mul`]. -#[inline] -pub(crate) fn p256_square(a: &[u64; 4]) -> Option<[u64; 4]> { - hosted!(x86_64::p256_square(a), { - let _ = a; - None - }) +/// `computation` over P-256's field arithmetic on the CPU's kernels where it +/// has them, otherwise `P256Field`'s own. The dispatch is per computation: +/// no field operation inside it checks the CPU. +#[inline(always)] +pub(crate) fn p256>(computation: C) -> C::Output { + hosted!(x86_64::p256(computation), computation.run::()) } /// a b mod 2^255 - 19 in `field25519`'s four-word form, by an instruction diff --git a/lib/crypto/src/arch/x86_64.rs b/lib/crypto/src/arch/x86_64.rs index 2e1839c79ab..0b9d3fc33aa 100644 --- a/lib/crypto/src/arch/x86_64.rs +++ b/lib/crypto/src/arch/x86_64.rs @@ -181,27 +181,65 @@ pub(super) fn rsa_mul(a: &[u64], b: &[u64], m: &[u64], n0: u64, n: usize, out: & Some(unsafe { montgomery::mul(a, b, m, n0, n, out) }) } -/// P-256 field multiplication by `p256::mul` where BMI2 and ADX are -/// admitted. +/// P-256's field arithmetic on `p256`'s kernels. The type is private to this +/// module and named only by [`p256`] once CPUID admitted BMI2 and ADX: that +/// is what makes its safe operations sound. #[cfg(not(target_os = "uefi"))] -#[inline] -pub(super) fn p256_mul(a: &[u64; 4], b: &[u64; 4]) -> Option<[u64; 4]> { - if !features().bmi2_adx() { - return None; +struct P256Kernels; + +#[cfg(not(target_os = "uefi"))] +impl crate::montgomery::Modulus<4> for P256Kernels { + const P: [u64; 4] = >::P; + const FORM: crate::montgomery::Form = crate::montgomery::Form::P256; + #[inline(always)] + fn add(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + p256::add(a, b) + } + #[inline(always)] + fn double(a: &[u64; 4]) -> [u64; 4] { + p256::double(a) + } + #[inline(always)] + fn sub(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + p256::sub(a, b) + } + #[inline(always)] + fn mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + // SAFETY: the type is only named once CPUID admitted BMI2 and ADX. + unsafe { p256::mul(a, b) } + } + #[inline(always)] + fn square(a: &[u64; 4]) -> [u64; 4] { + // SAFETY: as `mul`. + unsafe { p256::square(a) } + } + #[inline(always)] + fn select(table: &[T; 16], words: impl Fn(&T) -> [&[u64; 4]; W], index: u64) -> [[u64; 4]; W] { + if features().avx2() { + // SAFETY: CPUID and XCR0 admitted AVX2. + return unsafe { p256::select(table, words, index) }; + } + crate::montgomery::select(table, words, index) } - // CPUID admitted BMI2 and ADX. - Some(unsafe { p256::mul(a, b) }) } -/// P-256 field squaring by `p256::square`, as [`p256_mul`]. +/// `computation` over [`P256Kernels`] where CPUID admits BMI2 and ADX. #[cfg(not(target_os = "uefi"))] -#[inline] -pub(super) fn p256_square(a: &[u64; 4]) -> Option<[u64; 4]> { - if !features().bmi2_adx() { - return None; +#[inline(always)] +pub(super) fn p256>(computation: C) -> C::Output { + if features().bmi2_adx() { + // SAFETY: CPUID admitted BMI2 and ADX. + return unsafe { p256_kernels(computation) }; } - // CPUID admitted BMI2 and ADX. - Some(unsafe { p256::square(a) }) + computation.run::() +} + +/// # Safety +/// The CPU has BMI2 and ADX. +#[cfg(not(target_os = "uefi"))] +#[target_feature(enable = "bmi2,adx")] +unsafe fn p256_kernels>(computation: C) -> C::Output { + computation.run::() } /// 2^255 - 19 multiplication by `f25519::mul` where BMI2 and ADX are @@ -588,6 +626,45 @@ mod tests { } } + /// The AVX2 masked read against the portable one, for every index in + /// the table, either side of it and far past it. + #[test] + fn the_avx2_table_read_matches_the_portable_one() { + assert!( + features().avx2() || cfg!(feature = "portable"), + "this host's CPU has no AVX2, so nothing here can test the vector table read" + ); + if !features().avx2() { + return; + } + let table: [[[u64; 4]; 3]; 16] = + core::array::from_fn(|j| core::array::from_fn(|w| core::array::from_fn(|i| ((j * 12 + w * 4 + i) as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15)))); + for index in (0..18).chain([u64::MAX, 1 << 63]) { + let portable = crate::montgomery::select(&table, |e| [&e[0], &e[1], &e[2]], index); + // CPUID and XCR0 admitted AVX2. + assert_eq!(unsafe { p256::select(&table, |e| [&e[0], &e[1], &e[2]], index) }, portable, "index {index}"); + assert_eq!(portable, table[if index < 16 { index as usize } else { 0 }], "index {index}"); + } + } + + /// P-256 computations run on the kernels exactly where CPUID admits + /// BMI2 and ADX. + #[test] + fn p256_computations_run_on_the_kernels_the_cpu_admits() { + struct Arithmetic; + impl crate::montgomery::Computation<4> for Arithmetic { + type Output = &'static str; + fn run>(self) -> &'static str { + core::any::type_name::() + } + } + let expected = match features().bmi2_adx() { + true => core::any::type_name::(), + false => core::any::type_name::(), + }; + assert_eq!(p256(Arithmetic), expected); + } + #[test] fn features_match_host_detection_and_portable_override() { let f = features(); diff --git a/lib/crypto/src/arch/x86_64/p256.rs b/lib/crypto/src/arch/x86_64/p256.rs index 263e06f0e02..a3ad944e518 100644 --- a/lib/crypto/src/arch/x86_64/p256.rs +++ b/lib/crypto/src/arch/x86_64/p256.rs @@ -1,202 +1,229 @@ -//! P-256 field multiplication and squaring on BMI2 and ADX (Intel SDM Vol. -//! 2A, ADCX, ADOX and MULX). No branch or address depends on the operands. +//! P-256 field arithmetic in general registers: multiplication and squaring +//! on BMI2 and ADX (Intel SDM Vol. 2A, ADCX, ADOX and MULX), and addition and +//! subtraction on the base instruction set. Operands below p arrive in +//! registers, but for three words of multiplication's second, and results +//! below p leave in them, so a point formula's values need not pass through +//! memory between operations. No branch or address depends on the operands; +//! every conditional subtraction is a `cmov` or a mask taken from a borrow. +//! +//! Each Montgomery reduction step adds q p for q the lowest word, which +//! clears it as -p^-1 = 1 mod 2^64, and drops it: q 2^32 at the next word +//! (from q (2^64 - 1) + q (2^32 - 1) 2^64), and q p3 two words above that +//! (Gueron and Krasnov, "Fast prime field elliptic-curve cryptography with +//! 256-bit primes", J. Cryptogr. Eng. 2015). Squaring multiplies by p3 with +//! `mulx`; multiplication has no register left for p3 and forms +//! q p3 = q 2^64 - q 2^32 + q by shifts and subtraction. use core::arch::asm; - -/// p's top word; its other words enter the reduction step as shifts and -/// immediates. -const P3: u64 = >::P[3]; +use core::arch::x86_64::*; /// a b / 2^256 mod p for a, b < p, by the portable field's CIOS with P-256's -/// reduction step (`montgomery`'s `Form::P256`). +/// reduction step (`montgomery`'s `Form::P256`): a in registers, b0 in RDX +/// and b's other words read where they lie, one a row, so a b the caller +/// has just stored is not awaited before row 1. A template has 13 general +/// registers besides RDX, too few for both operands and the running words. /// -/// Row i adds a_i b to the five running words, carrying into a sixth: low -/// halves on CF through `adcx`, high halves on OF through `adox`. Its -/// reduction step, with q the low word (-p^-1 = 1 mod 2^64), adds q 2^96 as -/// q << 32 at word one and q >> 32 at word two, and q p3 at words three and -/// four, which leaves word zero zero and drops it. Each row's six words +/// Row i adds a b_i to the running words, carrying into a sixth: low halves +/// on CF through `adcx`, high halves on OF through `adox`. Each row's +/// reduction step leaves the low word zero and drops it, so the six words /// rotate one register on. The sum is below 2p, and p is subtracted where -/// the subtraction does not borrow, by `cmov`. +/// the subtraction does not borrow. /// /// # Safety /// The CPU has BMI2 and ADX. -#[inline] +#[inline(always)] pub(super) unsafe fn mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { - let (r0, r1, r2, r3): (u64, u64, u64, u64); - // SAFETY: the template reads the four words of `a` and of `b`, writes - // only its declared registers and leaves the stack alone. + let [mut r0, mut r1, mut r2, mut r3] = *a; + // SAFETY: the template reads b's words 1 to 3, writes only its declared + // registers and leaves the stack alone. unsafe { asm!( // Row 0 into (A B C D E), F = 0. - "mov rdx, [{a}]", - "mulx {B}, {A}, [{b}]", - "mulx {C}, {lo}, [{b} + 8]", + "mulx {B}, {A}, {a0}", + "mulx {C}, {lo}, {a1}", "add {B}, {lo}", - "mulx {D}, {lo}, [{b} + 16]", + "mulx {D}, {lo}, {a2}", "adc {C}, {lo}", - "mulx {E}, {lo}, [{b} + 24]", + "mulx {E}, {lo}, {a3}", "adc {D}, {lo}", "adc {E}, 0", "xor {F:e}, {F:e}", - // Reduction: q = A; the sum moves to (B C D E F). + // Reduction: q = A; q p3 into (A, rdx); the sum moves to (B C D E F). + "mov {lo}, {A}", + "shl {lo}, 32", + "mov {hi}, {A}", + "shr {hi}, 32", "mov rdx, {A}", - "mulx {hi}, {lo}, {p3}", - "shl rdx, 32", - "shr {A}, 32", - "add {B}, rdx", - "adc {C}, {A}", - "adc {D}, {lo}", - "adc {E}, {hi}", + "sub {A}, {lo}", + "sbb rdx, {hi}", + "add {B}, {lo}", + "adc {C}, {hi}", + "adc {D}, {A}", + "adc {E}, rdx", "adc {F}, 0", // Row 1 into (B C D E F), A = 0. - "mov rdx, [{a} + 8]", + "mov rdx, [{b} + 8]", "xor {A:e}, {A:e}", - "mulx {hi}, {lo}, [{b}]", + "mulx {hi}, {lo}, {a0}", "adcx {B}, {lo}", "adox {C}, {hi}", - "mulx {hi}, {lo}, [{b} + 8]", + "mulx {hi}, {lo}, {a1}", "adcx {C}, {lo}", "adox {D}, {hi}", - "mulx {hi}, {lo}, [{b} + 16]", + "mulx {hi}, {lo}, {a2}", "adcx {D}, {lo}", "adox {E}, {hi}", - "mulx {hi}, {lo}, [{b} + 24]", + "mulx {hi}, {lo}, {a3}", "adcx {E}, {lo}", "adox {F}, {hi}", "adcx {F}, {A}", "adox {A}, {A}", "adc {A}, 0", // Reduction: q = B; the sum moves to (C D E F A). + "mov {lo}, {B}", + "shl {lo}, 32", + "mov {hi}, {B}", + "shr {hi}, 32", "mov rdx, {B}", - "mulx {hi}, {lo}, {p3}", - "shl rdx, 32", - "shr {B}, 32", - "add {C}, rdx", - "adc {D}, {B}", - "adc {E}, {lo}", - "adc {F}, {hi}", + "sub {B}, {lo}", + "sbb rdx, {hi}", + "add {C}, {lo}", + "adc {D}, {hi}", + "adc {E}, {B}", + "adc {F}, rdx", "adc {A}, 0", // Row 2 into (C D E F A), B = 0. - "mov rdx, [{a} + 16]", + "mov rdx, [{b} + 16]", "xor {B:e}, {B:e}", - "mulx {hi}, {lo}, [{b}]", + "mulx {hi}, {lo}, {a0}", "adcx {C}, {lo}", "adox {D}, {hi}", - "mulx {hi}, {lo}, [{b} + 8]", + "mulx {hi}, {lo}, {a1}", "adcx {D}, {lo}", "adox {E}, {hi}", - "mulx {hi}, {lo}, [{b} + 16]", + "mulx {hi}, {lo}, {a2}", "adcx {E}, {lo}", "adox {F}, {hi}", - "mulx {hi}, {lo}, [{b} + 24]", + "mulx {hi}, {lo}, {a3}", "adcx {F}, {lo}", "adox {A}, {hi}", "adcx {A}, {B}", "adox {B}, {B}", "adc {B}, 0", // Reduction: q = C; the sum moves to (D E F A B). + "mov {lo}, {C}", + "shl {lo}, 32", + "mov {hi}, {C}", + "shr {hi}, 32", "mov rdx, {C}", - "mulx {hi}, {lo}, {p3}", - "shl rdx, 32", - "shr {C}, 32", - "add {D}, rdx", - "adc {E}, {C}", - "adc {F}, {lo}", - "adc {A}, {hi}", + "sub {C}, {lo}", + "sbb rdx, {hi}", + "add {D}, {lo}", + "adc {E}, {hi}", + "adc {F}, {C}", + "adc {A}, rdx", "adc {B}, 0", // Row 3 into (D E F A B), C = 0. - "mov rdx, [{a} + 24]", + "mov rdx, [{b} + 24]", "xor {C:e}, {C:e}", - "mulx {hi}, {lo}, [{b}]", + "mulx {hi}, {lo}, {a0}", "adcx {D}, {lo}", "adox {E}, {hi}", - "mulx {hi}, {lo}, [{b} + 8]", + "mulx {hi}, {lo}, {a1}", "adcx {E}, {lo}", "adox {F}, {hi}", - "mulx {hi}, {lo}, [{b} + 16]", + "mulx {hi}, {lo}, {a2}", "adcx {F}, {lo}", "adox {A}, {hi}", - "mulx {hi}, {lo}, [{b} + 24]", + "mulx {hi}, {lo}, {a3}", "adcx {A}, {lo}", "adox {B}, {hi}", "adcx {B}, {C}", "adox {C}, {C}", "adc {C}, 0", // Reduction: q = D; the sum moves to (E F A B C). + "mov {lo}, {D}", + "shl {lo}, 32", + "mov {hi}, {D}", + "shr {hi}, 32", "mov rdx, {D}", - "mulx {hi}, {lo}, {p3}", - "shl rdx, 32", - "shr {D}, 32", - "add {E}, rdx", - "adc {F}, {D}", - "adc {A}, {lo}", - "adc {B}, {hi}", + "sub {D}, {lo}", + "sbb rdx, {hi}", + "add {E}, {lo}", + "adc {F}, {hi}", + "adc {A}, {D}", + "adc {B}, rdx", "adc {C}, 0", - // (E F A B C) - p, p1 = 2^32 - 1 in b's register; no borrow out - // means the sum was at least p. - "mov {D}, {E}", - "mov {lo}, {F}", - "mov {hi}, {A}", - "mov rdx, {B}", - "mov {b:e}, 0xffffffff", - "sub {D}, -1", - "sbb {lo}, {b}", - "sbb {hi}, 0", - "sbb rdx, {p3}", + // (E F A B C) - p into a's registers; a borrow means the sum + // was below p, and it stands. + "mov {a0}, {E}", + "mov {a1}, {F}", + "mov {a2}, {A}", + "mov {a3}, {B}", + "mov {lo:e}, 0xffffffff", + "mov {hi}, 0xffffffff00000001", + "sub {a0}, -1", + "sbb {a1}, {lo}", + "sbb {a2}, 0", + "sbb {a3}, {hi}", "sbb {C}, 0", - "cmovnc {E}, {D}", - "cmovnc {F}, {lo}", - "cmovnc {A}, {hi}", - "cmovnc {B}, rdx", - a = in(reg) a.as_ptr(), - b = inout(reg) b.as_ptr() => _, - p3 = in(reg) P3, - A = out(reg) r2, - B = out(reg) r3, + "cmovc {a0}, {E}", + "cmovc {a1}, {F}", + "cmovc {a2}, {A}", + "cmovc {a3}, {B}", + a0 = inout(reg) r0, + a1 = inout(reg) r1, + a2 = inout(reg) r2, + a3 = inout(reg) r3, + b = in(reg) b.as_ptr(), + A = out(reg) _, + B = out(reg) _, C = out(reg) _, D = out(reg) _, - E = out(reg) r0, - F = out(reg) r1, + E = out(reg) _, + F = out(reg) _, lo = out(reg) _, hi = out(reg) _, - out("rdx") _, + inout("rdx") b[0] => _, options(pure, readonly, nostack), ); } [r0, r1, r2, r3] } -/// a^2 / 2^256 mod p for a < p: the cross products once, doubled, the -/// squares added, then [`mul`]'s reduction steps in place, the word each -/// step clears gathering the carry out of the top. +/// a^2 / 2^256 mod p for a < p in registers: the cross products once, +/// doubled, the squares added, then four reduction steps in place, the word +/// each step clears gathering the carry out of the top. Each step's q p3 is +/// a `mulx`, with p3 in a1's register once a1 is squared. /// /// # Safety /// The CPU has BMI2 and ADX. -#[inline] +#[inline(always)] pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] { + let [a0, a1, a2, a3] = *a; let (r0, r1, r2, r3): (u64, u64, u64, u64); - // SAFETY: the template reads the four words of `a`, writes only its - // declared registers and leaves the stack alone. + // SAFETY: the template writes only its declared registers and leaves + // memory and the stack alone. unsafe { asm!( // Cross products into (T1 .. T6). - "mov rdx, [{a}]", - "mulx {T2}, {T1}, [{a} + 8]", - "mulx {T3}, {lo}, [{a} + 16]", + "mov rdx, {a0}", + "mulx {T2}, {T1}, {a1}", + "mulx {T3}, {lo}, {a2}", "add {T2}, {lo}", - "mulx {T4}, {lo}, [{a} + 24]", + "mulx {T4}, {lo}, {a3}", "adc {T3}, {lo}", "adc {T4}, 0", - "mov rdx, [{a} + 8]", - "mulx {hi}, {lo}, [{a} + 16]", + "mov rdx, {a1}", + "mulx {hi}, {lo}, {a2}", "add {T3}, {lo}", "adc {T4}, {hi}", - "mulx {T5}, {lo}, [{a} + 24]", + "mulx {T5}, {lo}, {a3}", "adc {T5}, 0", "add {T4}, {lo}", "adc {T5}, 0", - "mov rdx, [{a} + 16]", - "mulx {T6}, {lo}, [{a} + 24]", + "mov rdx, {a2}", + "mulx {T6}, {lo}, {a3}", "add {T5}, {lo}", "adc {T6}, 0", // Doubled into (T1 .. T7). @@ -208,39 +235,39 @@ pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] { "adc {T5}, {T5}", "adc {T6}, {T6}", "adc {T7}, 0", - // The squares added into (T0 .. T7). - "mov rdx, [{a}]", - "mulx {hi}, {T0}, rdx", + // The squares added into (T0 .. T7), T0 in a0's register. + "mov rdx, {a0}", + "mulx {hi}, {a0}, rdx", "add {T1}, {hi}", - "mov rdx, [{a} + 8]", + "mov rdx, {a1}", "mulx {hi}, {lo}, rdx", "adc {T2}, {lo}", "adc {T3}, {hi}", - "mov rdx, [{a} + 16]", + "mov rdx, {a2}", "mulx {hi}, {lo}, rdx", "adc {T4}, {lo}", "adc {T5}, {hi}", - "mov rdx, [{a} + 24]", + "mov rdx, {a3}", "mulx {hi}, {lo}, rdx", "adc {T6}, {lo}", "adc {T7}, {hi}", + "mov {a1}, 0xffffffff00000001", // Reduction steps; T0 then gathers the carry out of T7. - "mov {a}, {p3}", - "mov rdx, {T0}", - "mulx {hi}, {lo}, {a}", + "mov rdx, {a0}", + "mulx {hi}, {lo}, {a1}", "shl rdx, 32", - "shr {T0}, 32", + "shr {a0}, 32", "add {T1}, rdx", - "adc {T2}, {T0}", + "adc {T2}, {a0}", "adc {T3}, {lo}", "adc {T4}, {hi}", "adc {T5}, 0", "adc {T6}, 0", "adc {T7}, 0", - "mov {T0:e}, 0", - "adc {T0}, 0", + "mov {a0:e}, 0", + "adc {a0}, 0", "mov rdx, {T1}", - "mulx {hi}, {lo}, {a}", + "mulx {hi}, {lo}, {a1}", "shl rdx, 32", "shr {T1}, 32", "add {T2}, rdx", @@ -249,9 +276,9 @@ pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] { "adc {T5}, {hi}", "adc {T6}, 0", "adc {T7}, 0", - "adc {T0}, 0", + "adc {a0}, 0", "mov rdx, {T2}", - "mulx {hi}, {lo}, {a}", + "mulx {hi}, {lo}, {a1}", "shl rdx, 32", "shr {T2}, 32", "add {T3}, rdx", @@ -259,34 +286,35 @@ pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] { "adc {T5}, {lo}", "adc {T6}, {hi}", "adc {T7}, 0", - "adc {T0}, 0", + "adc {a0}, 0", "mov rdx, {T3}", - "mulx {hi}, {lo}, {a}", + "mulx {hi}, {lo}, {a1}", "shl rdx, 32", "shr {T3}, 32", "add {T4}, rdx", "adc {T5}, {T3}", "adc {T6}, {lo}", "adc {T7}, {hi}", - "adc {T0}, 0", - // (T4 .. T7 T0) - p, as in `mul`. - "mov {T1}, {T4}", - "mov {T2}, {T5}", - "mov {T3}, {T6}", + "adc {a0}, 0", + // (T4 .. T7 a0) - p, as in `mul`. + "mov {a2}, {T4}", + "mov {a3}, {T5}", + "mov {T1}, {T6}", "mov rdx, {T7}", "mov {lo:e}, 0xffffffff", - "sub {T1}, -1", - "sbb {T2}, {lo}", - "sbb {T3}, 0", - "sbb rdx, {a}", - "sbb {T0}, 0", - "cmovnc {T4}, {T1}", - "cmovnc {T5}, {T2}", - "cmovnc {T6}, {T3}", + "sub {a2}, -1", + "sbb {a3}, {lo}", + "sbb {T1}, 0", + "sbb rdx, {a1}", + "sbb {a0}, 0", + "cmovnc {T4}, {a2}", + "cmovnc {T5}, {a3}", + "cmovnc {T6}, {T1}", "cmovnc {T7}, rdx", - a = inout(reg) a.as_ptr() => _, - p3 = const P3, - T0 = out(reg) _, + a0 = inout(reg) a0 => _, + a1 = inout(reg) a1 => _, + a2 = inout(reg) a2 => _, + a3 = inout(reg) a3 => _, T1 = out(reg) _, T2 = out(reg) _, T3 = out(reg) _, @@ -297,8 +325,183 @@ pub(super) unsafe fn square(a: &[u64; 4]) -> [u64; 4] { lo = out(reg) _, hi = out(reg) _, out("rdx") _, - options(pure, readonly, nostack), + options(pure, nomem, nostack), + ); + } + [r0, r1, r2, r3] +} + +/// (a + b) mod p for a, b < p: the five-word sum, then p subtracted where +/// that does not borrow. +#[inline(always)] +pub(super) fn add(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + let [mut r0, mut r1, mut r2, mut r3] = *a; + let [b0, b1, b2, b3] = *b; + // SAFETY: the template writes only its declared registers and leaves + // memory and the stack alone. + unsafe { + asm!( + "xor {c:e}, {c:e}", + "add {r0}, {b0}", + "adc {r1}, {b1}", + "adc {r2}, {b2}", + "adc {r3}, {b3}", + "adc {c}, 0", + "mov {b0}, {r0}", + "mov {b1}, {r1}", + "mov {b2}, {r2}", + "mov {b3}, {r3}", + "mov {p1:e}, 0xffffffff", + "mov {p3}, 0xffffffff00000001", + "sub {b0}, -1", + "sbb {b1}, {p1}", + "sbb {b2}, 0", + "sbb {b3}, {p3}", + "sbb {c}, 0", + "cmovnc {r0}, {b0}", + "cmovnc {r1}, {b1}", + "cmovnc {r2}, {b2}", + "cmovnc {r3}, {b3}", + r0 = inout(reg) r0, + r1 = inout(reg) r1, + r2 = inout(reg) r2, + r3 = inout(reg) r3, + b0 = inout(reg) b0 => _, + b1 = inout(reg) b1 => _, + b2 = inout(reg) b2 => _, + b3 = inout(reg) b3 => _, + c = out(reg) _, + p1 = out(reg) _, + p3 = out(reg) _, + options(pure, nomem, nostack), + ); + } + [r0, r1, r2, r3] +} + +/// 2a mod p for a < p, as [`add`]. +#[inline(always)] +pub(super) fn double(a: &[u64; 4]) -> [u64; 4] { + let [mut r0, mut r1, mut r2, mut r3] = *a; + // SAFETY: the template writes only its declared registers and leaves + // memory and the stack alone. + unsafe { + asm!( + "xor {c:e}, {c:e}", + "add {r0}, {r0}", + "adc {r1}, {r1}", + "adc {r2}, {r2}", + "adc {r3}, {r3}", + "adc {c}, 0", + "mov {t0}, {r0}", + "mov {t1}, {r1}", + "mov {t2}, {r2}", + "mov {t3}, {r3}", + "mov {p1:e}, 0xffffffff", + "mov {p3}, 0xffffffff00000001", + "sub {t0}, -1", + "sbb {t1}, {p1}", + "sbb {t2}, 0", + "sbb {t3}, {p3}", + "sbb {c}, 0", + "cmovnc {r0}, {t0}", + "cmovnc {r1}, {t1}", + "cmovnc {r2}, {t2}", + "cmovnc {r3}, {t3}", + r0 = inout(reg) r0, + r1 = inout(reg) r1, + r2 = inout(reg) r2, + r3 = inout(reg) r3, + t0 = out(reg) _, + t1 = out(reg) _, + t2 = out(reg) _, + t3 = out(reg) _, + c = out(reg) _, + p1 = out(reg) _, + p3 = out(reg) _, + options(pure, nomem, nostack), + ); + } + [r0, r1, r2, r3] +} + +/// (a - b) mod p for a, b < p: the difference, then p added under the mask +/// of its borrow. p's words under a mask m are m, m >> 32, 0 and +/// (m << 32) - m. +#[inline(always)] +pub(super) fn sub(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + let [mut r0, mut r1, mut r2, mut r3] = *a; + let [b0, b1, b2, b3] = *b; + // SAFETY: the template writes only its declared registers and leaves + // memory and the stack alone. + unsafe { + asm!( + "sub {r0}, {b0}", + "sbb {r1}, {b1}", + "sbb {r2}, {b2}", + "sbb {r3}, {b3}", + "sbb {b0}, {b0}", + "mov {b1}, {b0}", + "shr {b1}, 32", + "mov {b3}, {b0}", + "shl {b3}, 32", + "sub {b3}, {b0}", + "add {r0}, {b0}", + "adc {r1}, {b1}", + "adc {r2}, 0", + "adc {r3}, {b3}", + r0 = inout(reg) r0, + r1 = inout(reg) r1, + r2 = inout(reg) r2, + r3 = inout(reg) r3, + b0 = inout(reg) b0 => _, + b1 = inout(reg) b1 => _, + b2 = in(reg) b2, + b3 = inout(reg) b3 => _, + options(pure, nomem, nostack), ); } [r0, r1, r2, r3] } + +/// The words of table[index], or table[0] for an index past 15, in AVX2 +/// registers: each entry's residues are one vector each, and each is kept or +/// replaced under the mask of a vector equality between the index and a +/// vector count of the entries. Every entry is read. +/// +/// # Safety +/// The CPU has AVX2 and the OS saves its state. +#[target_feature(enable = "avx2")] +#[allow(clippy::needless_range_loop, reason = "iterator adaptors stay out of line in the shipped profile")] +pub(super) unsafe fn select( + table: &[T; 16], + words: impl Fn(&T) -> [&[u64; 4]; W], + index: u64, +) -> [[u64; 4]; W] { + // `lddqu`, an unaligned load like `_mm256_loadu_si256`, which copies + // through the stack under debug assertions. + let load = |w: &[u64; 4]| -> __m256i { + // SAFETY: four words are 32 readable bytes. + unsafe { _mm256_lddqu_si256(w.as_ptr().cast()) } + }; + let index = _mm256_set1_epi64x(index as i64); + let first = words(&table[0]); + let mut chosen: [__m256i; W] = core::array::from_fn(|w| load(first[w])); + let one = _mm256_set1_epi64x(1); + let mut count = one; + for j in 1..16 { + let mut m = _mm256_cmpeq_epi64(index, count); + // Opaque, so LLVM keeps the logical select below rather than forming + // a `vblendvpd`, which is not one of the crate's audited + // instructions. + // SAFETY: the template reads and writes only the operand's register. + unsafe { asm!("/* {0} */", inout(ymm_reg) m, options(pure, nomem, nostack, preserves_flags)) }; + count = _mm256_add_epi64(count, one); + let entry = words(&table[j]); + for w in 0..W { + chosen[w] = _mm256_or_si256(_mm256_andnot_si256(m, chosen[w]), _mm256_and_si256(m, load(entry[w]))); + } + } + // SAFETY: a vector and four words are the same 32 plain bytes. + chosen.map(|v| unsafe { core::mem::transmute::<__m256i, [u64; 4]>(v) }) +} diff --git a/lib/crypto/src/chacha.rs b/lib/crypto/src/chacha.rs index 0e2803160e5..cc1799a8cab 100644 --- a/lib/crypto/src/chacha.rs +++ b/lib/crypto/src/chacha.rs @@ -131,6 +131,7 @@ fn stream(key: &[u8; 32], counter: u64, nonce: &[u8; 8], data: &mut [u8]) { /// RFC 8439 §2.4: the keystream from block `counter` of (`key`, `nonce`) /// XORed into `data`. `Length` when the blocks would pass 2^32 - 1. pub fn xor(key: &[u8; 32], nonce: &[u8; 12], counter: u32, data: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); check_ietf(data.len(), counter)?; let (counter, nonce) = ietf(nonce, counter); stream(key, counter, &nonce, data); diff --git a/lib/crypto/src/ct.rs b/lib/crypto/src/ct.rs index 3c70850dd0e..135579cf829 100644 --- a/lib/crypto/src/ct.rs +++ b/lib/crypto/src/ct.rs @@ -28,6 +28,7 @@ pub(crate) fn nonzero(value: u64) -> u64 { /// Whether `a` and `b` hold the same bytes, without branching or indexing on their contents: /// compare a received authenticator with the expected one through this. pub fn equal(a: &[u8], b: &[u8]) -> bool { + let _protection = crate::timing::protect(); if a.len() != b.len() { return false; } diff --git a/lib/crypto/src/ecc.rs b/lib/crypto/src/ecc.rs index db225e6ffdb..24ae31732c0 100644 --- a/lib/crypto/src/ecc.rs +++ b/lib/crypto/src/ecc.rs @@ -9,7 +9,7 @@ macro_rules! curve_api { use crate::{ Error, Random, hash::$hash, - nist::{$curve, Curve, Point, Scalar}, + nist::{$curve, Point, Scalar}, secret::Secret, }; pub struct PublicKey { @@ -45,23 +45,24 @@ macro_rules! curve_api { } impl SecretKey { pub fn from_bytes(bytes: [u8; $bytes]) -> Result { + let _protection = crate::timing::protect(); let bytes = Secret(bytes); Ok(Self(Scalar::from_bytes(&bytes.0)?)) } pub fn generate(random: &mut impl Random) -> Result { + let _protection = crate::timing::protect(); let mut candidates = Secret([0; 4 * $bytes]); random.fill(&mut candidates.0)?; Ok(Self(Scalar::first_valid(&candidates.0)?)) } pub fn public_key(&self) -> PublicKey { - // The base point's multiple by a nonzero scalar below n - // is finite and on the curve. - let point = <$curve as Curve<$limbs>>::base_mul(&self.0); + let _protection = crate::timing::protect(); let mut encoded = [0; $encoded]; - point.encode(&mut encoded).expect("nonzero prime-order scalar"); + let point = Point::public_key(&self.0, &mut encoded); PublicKey { point, encoded } } pub fn agree(self, peer: &PublicKey) -> Result { + let _protection = crate::timing::protect(); let mut out = Secret([0; $bytes]); peer.point.agree(&self.0, &mut out.0); Ok(SharedSecret(out)) diff --git a/lib/crypto/src/ed25519.rs b/lib/crypto/src/ed25519.rs index 33af0b1f2f2..56e7d09c750 100644 --- a/lib/crypto/src/ed25519.rs +++ b/lib/crypto/src/ed25519.rs @@ -597,6 +597,7 @@ pub struct SigningKey { #[cfg(feature = "sign")] impl SigningKey { pub fn from_seed(seed: [u8; 32]) -> Self { + let _protection = crate::timing::protect(); let seed = Secret(seed); let mut expanded = Secret(Sha512::digest(&seed.0).expect("fixed 32-byte seed")); expanded.0[0] &= 248; @@ -620,6 +621,7 @@ impl SigningKey { &self.public } pub fn sign(&self, message: &[u8]) -> Result<[u8; 64], Error> { + let _protection = crate::timing::protect(); let mut h = Sha512::new(); h.update(&self.prefix.0)?; h.update(message)?; diff --git a/lib/crypto/src/hybrid.rs b/lib/crypto/src/hybrid.rs index b81b4ac90e1..9aa8541920e 100644 --- a/lib/crypto/src/hybrid.rs +++ b/lib/crypto/src/hybrid.rs @@ -18,12 +18,14 @@ impl SshSecret { } } pub fn for_tls(kem: mlkem::SharedSecret, ecdh: x25519::SharedSecret) -> TlsSecret { + let _protection = crate::timing::protect(); let mut out = Secret([0; 64]); out.0[..32].copy_from_slice(kem.as_bytes()); out.0[32..].copy_from_slice(ecdh.as_bytes()); TlsSecret(out) } pub fn for_ssh(kem: mlkem::SharedSecret, ecdh: x25519::SharedSecret) -> SshSecret { + let _protection = crate::timing::protect(); let mut hash = Sha256::new(); hash.update(kem.as_bytes()) .expect("fixed-size shared secret"); diff --git a/lib/crypto/src/kdf.rs b/lib/crypto/src/kdf.rs index 949e4da312e..46086884bbd 100644 --- a/lib/crypto/src/kdf.rs +++ b/lib/crypto/src/kdf.rs @@ -57,20 +57,24 @@ impl Hmac { } #[inline(always)] pub fn new(key: &[u8]) -> Result { + let _protection = crate::timing::protect(); let mut mac = Self::unkeyed(); mac.key(key)?; Ok(mac) } #[inline] pub fn update(&mut self, message: &[u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); self.inner.update(message) } #[inline(always)] pub fn finalize(mut self, output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); self.finish(output) } #[inline(always)] pub fn authenticate(key: &[u8], message: &[u8], output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); let mut mac = Self::unkeyed(); mac.key(key)?; mac.update(message)?; @@ -78,6 +82,7 @@ impl Hmac { } #[inline(always)] pub fn verify(mut self, tag: &[u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); if tag.len() != H::OUTPUT_LEN { return Err(Error::Length); } @@ -100,6 +105,7 @@ pub struct Hkdf { impl Hkdf { #[inline] pub fn extract(salt: &[u8], input_key_material: &[u8]) -> Result { + let _protection = crate::timing::protect(); let mut prk = Secret([0u8; 64]); Hmac::::authenticate(salt, input_key_material, &mut prk.0[..H::OUTPUT_LEN])?; Ok(Self { @@ -121,6 +127,7 @@ impl Hkdf { } #[inline] pub fn expand(&self, info: &[&[u8]], output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); if output.len() > 255 * H::OUTPUT_LEN { return Err(Error::Length); } @@ -154,6 +161,7 @@ impl Hkdf { /// PBKDF2 (RFC 8018 §5.2) with HMAC over `H` as its PRF, filling `output`, /// which is 1 to (2^32 - 1) * `H::OUTPUT_LEN` bytes. pub fn pbkdf2(password: &[u8], salt: &[u8], iterations: NonZeroU32, output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); let keyed = Hmac::::new(password)?; let mut salted = keyed.fork(); salted.update(salt)?; diff --git a/lib/crypto/src/lib.rs b/lib/crypto/src/lib.rs index 1073b1ca4d9..33b26e6b6fc 100644 --- a/lib/crypto/src/lib.rs +++ b/lib/crypto/src/lib.rs @@ -1,8 +1,9 @@ //! Cryptographic primitives for TLS, SSH, boot verification, passphrase-derived //! keys and WPA2. //! -//! The core has no allocator, entropy source, runtime dependency or host build -//! tool. Inputs crossing an encoding boundary return [`Error`]; fixed-size +//! The core has no allocator, entropy source, host build tool or runtime +//! dependency but, on ToyOS, its timing protection (`timing`). Inputs +//! crossing an encoding boundary return [`Error`]; fixed-size //! representations hold the invariants inside that boundary. Message lengths, //! algorithm choices and CPU capabilities are public. Module contracts identify //! operations permitted to depend on public data. @@ -49,6 +50,7 @@ mod secret; pub use secret::{Secret, Zeroable}; pub mod siphash; pub mod ssh; +pub mod timing; #[cfg(feature = "wpa2")] pub mod wpa2; pub mod x25519; diff --git a/lib/crypto/src/mlkem.rs b/lib/crypto/src/mlkem.rs index f82611a9f60..c31480c549d 100644 --- a/lib/crypto/src/mlkem.rs +++ b/lib/crypto/src/mlkem.rs @@ -290,6 +290,7 @@ impl EncapsulationKey { &self, random: &mut impl Random, ) -> Result<([u8; 1088], SharedSecret), Error> { + let _protection = crate::timing::protect(); let mut message = Secret([0; 32]); random.fill(&mut message.0)?; let hash = keccak::h(&[&self.bytes]); @@ -324,6 +325,7 @@ impl EncapsulationKey { } impl DecapsulationKey { pub fn generate(random: &mut impl Random) -> Result { + let _protection = crate::timing::protect(); let mut seeds = Secret([0; 64]); random.fill(&mut seeds.0)?; let expanded = Secret(keccak::g(&[&seeds.0[..32], &[3]])); @@ -354,6 +356,7 @@ impl DecapsulationKey { /// Invalid, correctly sized ciphertexts return a pseudorandom shared secret. /// The calling protocol discovers failure through its key confirmation. pub fn decapsulate(self, ciphertext: &[u8]) -> Result { + let _protection = crate::timing::protect(); if ciphertext.len() != 1088 { return Err(Error::Length); } diff --git a/lib/crypto/src/montgomery.rs b/lib/crypto/src/montgomery.rs index 3803e795eb7..3fd1c289efe 100644 --- a/lib/crypto/src/montgomery.rs +++ b/lib/crypto/src/montgomery.rs @@ -1,8 +1,11 @@ //! Montgomery arithmetic modulo the NIST curves' primes and orders, N = 4 or 6 //! words. Each modulus is a type, so a residue modulo one cannot meet a residue -//! modulo another. Every operation on a residue has a fixed schedule: masked -//! final subtractions, and inversion by a fixed count of divsteps. Residues -//! are `Copy` intermediates, not erased (`secret`'s contract). +//! modulo another, and a computation names a second type of the same p only +//! to run on its arithmetic (`Modulus::dispatch`). Every operation on a +//! residue has a fixed schedule: masked final subtractions, and inversion by +//! a fixed count of divsteps; only `Residue::inverse_of_public`, for public +//! integers, does not. Residues are `Copy` intermediates, not erased +//! (`secret`'s contract). #![allow( clippy::needless_range_loop, reason = "carry chains index parallel fixed-size arrays; iterator adaptors stay out of line in the shipped profile" @@ -12,9 +15,11 @@ use core::marker::PhantomData; mod divsteps; -/// An odd modulus p with 2^(64N-1) < p < 2^(64N), and its Montgomery -/// constants for R = 2^(64N), evaluated at compile time. -pub(crate) trait Modulus: 'static { +/// An odd modulus p with 2^(64N-1) < p < 2^(64N), its Montgomery constants +/// for R = 2^(64N), evaluated at compile time, and the arithmetic on its +/// residues. Each operation takes and returns words below p, on a fixed +/// schedule; a modulus with a faster form of one overrides it. +pub(crate) trait Modulus: Sized + 'static { const P: [u64; N]; /// How a reduction step forms q p. const FORM: Form = Form::Generic; @@ -24,6 +29,66 @@ pub(crate) trait Modulus: 'static { const ONE: [u64; N] = shift_mod(&one(), &Self::P, 64 * N); /// R^2 mod p. const R2: [u64; N] = shift_mod(&one(), &Self::P, 128 * N); + + fn add(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + let mut v = [0; N]; + let mut carry = false; + for i in 0..N { + (v[i], carry) = carrying_add(a[i], b[i], carry); + } + reduce(&v, u64::from(carry), &Self::P) + } + #[inline(always)] + fn double(a: &[u64; N]) -> [u64; N] { + Self::add(a, a) + } + fn sub(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + let mut v = [0; N]; + let mut borrow = false; + for i in 0..N { + (v[i], borrow) = borrowing_sub(a[i], b[i], borrow); + } + let mask = ct::mask(u64::from(borrow)); + let mut carry = false; + for i in 0..N { + (v[i], carry) = carrying_add(v[i], Self::P[i] & mask, carry); + } + v + } + /// a b / R mod p, word by word with each product row followed by its + /// reduction step (CIOS; Koc, Acar and Kaliski, "Analyzing and comparing + /// Montgomery multiplication algorithms", IEEE Micro 1996). + fn mul(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + let (t, high) = product::(a, b); + reduce(&t, high, &Self::P) + } + /// a^2 / R mod p: the N(N - 1)/2 cross products once, doubled, the N + /// squares added, then N reduction steps over the 2N-word square. + fn square(a: &[u64; N]) -> [u64; N] { + let (t, high) = redc::(&mut square_wide(a)); + reduce(&t, high, &Self::P) + } + /// The words of table[index] for a secret index, or table[0] for an + /// index past 15, reading every entry under masks: the table's order + /// and length are public. + #[inline(always)] + fn select(table: &[T; 16], words: impl Fn(&T) -> [&[u64; N]; W], index: u64) -> [[u64; N]; W] { + select(table, words, index) + } + /// `computation` over the fastest arithmetic modulo p that the CPU + /// admits: this modulus's own unless it says otherwise. + #[inline(always)] + fn dispatch>(computation: C) -> C::Output { + computation.run::() + } +} + +/// A computation over residues modulo one p, generic over the modulus type +/// whose arithmetic computes them; [`Modulus::dispatch`] chooses it, and +/// every type it passes has the same p. +pub(crate) trait Computation { + type Output; + fn run>(self) -> Self::Output; } /// The shape of a modulus that its reduction step exploits. @@ -58,10 +123,14 @@ unsafe impl, const N: usize> crate::secret::Zeroable for Residue, const N: usize> Residue { /// Words already in Montgomery form and below p: a compile-time constant - /// from [`encode`]. + /// from [`encode`], or a residue's own [`Self::words`]. pub(crate) const fn from_montgomery(words: [u64; N]) -> Self { Self(words, PhantomData) } + /// The Montgomery-form words, for [`Modulus::select`]. + pub(crate) const fn words(&self) -> &[u64; N] { + &self.0 + } pub(crate) const fn zero() -> Self { Self([0; N], PhantomData) } @@ -88,73 +157,32 @@ impl, const N: usize> Residue { let one = Self(one(), PhantomData); self.mul(&one).0 } - /// P-256's operations inline into the point formulas; its portable - /// multiplication and squaring stay out of line. #[inline(always)] pub(crate) fn add(&self, b: &Self) -> Self { - if let Form::P256 = M::FORM { - return Self(p256::add(&self.0, &b.0), PhantomData); - } - self.add_generic(b) - } - fn add_generic(&self, b: &Self) -> Self { - let mut v = [0; N]; - let mut carry = false; - for i in 0..N { - (v[i], carry) = carrying_add(self.0[i], b.0[i], carry); - } - Self(reduce(&v, u64::from(carry), &M::P), PhantomData) + Self(M::add(&self.0, &b.0), PhantomData) } #[inline(always)] pub(crate) fn double(&self) -> Self { - self.add(self) + Self(M::double(&self.0), PhantomData) } #[inline(always)] pub(crate) fn sub(&self, b: &Self) -> Self { - if let Form::P256 = M::FORM { - return Self(p256::sub(&self.0, &b.0), PhantomData); - } - self.sub_generic(b) - } - fn sub_generic(&self, b: &Self) -> Self { - let mut v = [0; N]; - let mut borrow = false; - for i in 0..N { - (v[i], borrow) = borrowing_sub(self.0[i], b.0[i], borrow); - } - let mask = ct::mask(u64::from(borrow)); - let mut carry = false; - for i in 0..N { - (v[i], carry) = carrying_add(v[i], M::P[i] & mask, carry); - } - Self(v, PhantomData) + Self(M::sub(&self.0, &b.0), PhantomData) } - /// self * b / R mod p, word by word with each product row followed by its - /// reduction step (CIOS; Koc, Acar and Kaliski, "Analyzing and comparing - /// Montgomery multiplication algorithms", IEEE Micro 1996). #[inline(always)] pub(crate) fn mul(&self, b: &Self) -> Self { - if let Form::P256 = M::FORM { - return Self(p256::mul(&self.0, &b.0), PhantomData); - } - self.mul_generic(b) - } - fn mul_generic(&self, b: &Self) -> Self { - let (t, high) = product::(&self.0, &b.0); - Self(reduce(&t, high, &M::P), PhantomData) + Self(M::mul(&self.0, &b.0), PhantomData) } - /// self^2 / R mod p: the N(N - 1)/2 cross products once, doubled, the N - /// squares added, then N reduction steps over the 2N-word square. #[inline(always)] pub(crate) fn square(&self) -> Self { - if let Form::P256 = M::FORM { - return Self(p256::square(&self.0), PhantomData); - } - self.square_generic() + Self(M::square(&self.0), PhantomData) } - fn square_generic(&self) -> Self { - let (t, high) = redc::(&mut square_wide(&self.0)); - Self(reduce(&t, high, &M::P), PhantomData) + /// The same residue under another modulus type with the same p: the + /// type whose arithmetic [`Modulus::dispatch`] chose. + #[inline(always)] + pub(crate) fn cast>(&self) -> Residue { + const { assert!(same(&M::P, &F::P)) }; + Residue(self.0, PhantomData) } /// The `const` forms of [`Self::mul`], [`Self::add`] and [`Self::sub`], /// for tables built at compile time and for tests: not for secrets. They @@ -247,6 +275,11 @@ impl, const N: usize> Residue { // (x R)^-1 R^2 = x^-1 R. Self(divsteps::invert::(&self.0, &M::R2), PhantomData) } + /// The residue x^-1 for a public integer 0 < x < p, in variable time. + pub(crate) fn inverse_of_public(x: &[u64; N]) -> Self { + // x^-1 R is x^-1's Montgomery form. + Self(divsteps::invert_public::(x, &M::ONE), PhantomData) + } } /// p - 2 for M's p. @@ -262,12 +295,10 @@ const fn exponent, const N: usize>() -> [u64; N] { e } -/// P-256's field arithmetic written out word by word: the shipped profile -/// does not unroll the generic loops, and P-256 is the hot curve. Every step -/// is the generic one's (`product`, `square_wide`, `redc`, `reduce`). -/// Multiplication and squaring take the instruction paths where the CPU has -/// them (`arch::p256_mul`, `arch::p256_square`). -mod p256 { +/// P-256's field arithmetic written out word by word, `P256Field`'s: the +/// shipped profile does not unroll the generic loops. Every step is the +/// generic one's (`product`, `square_wide`, `redc`, `reduce`). +pub(crate) mod p256 { use super::{Form, Modulus, ct, mac}; use crate::nist::P256Field; @@ -288,37 +319,19 @@ mod p256 { (d, (b1 | b2) as u64) } - /// The four words as a residue's array, whose N is four here; indexing - /// rather than a slice copy keeps the words in registers. - #[inline(always)] - fn words(w: [u64; 4]) -> [u64; N] { - let mut out = [0; N]; - out[0] = w[0]; - out[1] = w[1]; - out[2] = w[2]; - out[3] = w[3]; - out - } - - /// A residue's array as four words, which N is here. - #[inline(always)] - fn four(a: &[u64; N]) -> &[u64; 4] { - a.as_slice().try_into().expect("P-256 residues are four words") - } - /// v mod p for v + high 2^256 < 2p, p subtracted under a mask. #[inline(always)] - fn reduce(v: [u64; 4], high: u64) -> [u64; N] { + fn reduce(v: [u64; 4], high: u64) -> [u64; 4] { let (d0, b) = sbb(v[0], P[0], 0); let (d1, b) = sbb(v[1], P[1], b); let (d2, b) = sbb(v[2], P[2], b); let (d3, b) = sbb(v[3], P[3], b); let m = ct::mask(ct::nonzero(high) | (1 ^ b)); - words([(v[0] & !m) | (d0 & m), (v[1] & !m) | (d1 & m), (v[2] & !m) | (d2 & m), (v[3] & !m) | (d3 & m)]) + [(v[0] & !m) | (d0 & m), (v[1] & !m) | (d1 & m), (v[2] & !m) | (d2 & m), (v[3] & !m) | (d3 & m)] } #[inline(always)] - pub(super) fn add(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + pub(crate) fn add(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { let (v0, c) = adc(a[0], b[0], 0); let (v1, c) = adc(a[1], b[1], c); let (v2, c) = adc(a[2], b[2], c); @@ -327,7 +340,7 @@ mod p256 { } #[inline(always)] - pub(super) fn sub(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + pub(crate) fn sub(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { let (v0, w) = sbb(a[0], b[0], 0); let (v1, w) = sbb(a[1], b[1], w); let (v2, w) = sbb(a[2], b[2], w); @@ -337,20 +350,13 @@ mod p256 { let (r1, c) = adc(v1, P[1] & m, c); let (r2, c) = adc(v2, P[2] & m, c); let (r3, _) = adc(v3, P[3] & m, c); - words([r0, r1, r2, r3]) + [r0, r1, r2, r3] } - #[inline(always)] - pub(super) fn mul(a: &[u64; N], b: &[u64; N]) -> [u64; N] { - match crate::arch::p256_mul(four(a), four(b)) { - Some(v) => words(v), - None => mul_portable(a, b), - } - } - - /// CIOS with P-256's reduction step, as `product`. + /// CIOS with P-256's reduction step, as `product`; out of line in the + /// point formulas. #[inline(never)] - fn mul_portable(a: &[u64; N], b: &[u64; N]) -> [u64; N] { + pub(crate) fn mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { let (mut t0, mut t1, mut t2, mut t3, mut high) = (0u64, 0u64, 0u64, 0u64, 0u64); macro_rules! row { ($ai:expr) => {{ @@ -376,18 +382,10 @@ mod p256 { reduce([t0, t1, t2, t3], high) } - #[inline(always)] - pub(super) fn square(a: &[u64; N]) -> [u64; N] { - match crate::arch::p256_square(four(a)) { - Some(v) => words(v), - None => square_portable(a), - } - } - /// The cross products once, doubled, the squares added, then four - /// reduction steps, as `square_wide` and `redc`. + /// reduction steps, as `square_wide` and `redc`; out of line. #[inline(never)] - fn square_portable(a: &[u64; N]) -> [u64; N] { + pub(crate) fn square(a: &[u64; 4]) -> [u64; 4] { let (t1, c) = mac(0, a[0], a[1], 0); let (t2, c) = mac(0, a[0], a[2], c); let (t3, t4) = mac(0, a[0], a[3], c); @@ -438,6 +436,40 @@ mod p256 { } } +/// [`Modulus::select`]'s portable form, by `ct` masks. +#[inline(always)] +pub(crate) fn select( + table: &[T; 16], + words: impl Fn(&T) -> [&[u64; N]; W], + index: u64, +) -> [[u64; N]; W] { + let mut chosen = words(&table[0]).map(|w| *w); + // Indexing, not an iterator adaptor, which the shipped profile keeps out + // of line. + for j in 1..16 { + let m = ct::mask(1 ^ ct::nonzero(index ^ j as u64)); + let entry = words(&table[j]); + for w in 0..W { + for i in 0..N { + chosen[w][i] = (chosen[w][i] & !m) | (entry[w][i] & m); + } + } + } + chosen +} + +/// Whether two moduli are one p, at compile time. +const fn same(a: &[u64; N], b: &[u64; N]) -> bool { + let mut i = 0; + while i < N { + if a[i] != b[i] { + return false; + } + i += 1; + } + true +} + /// One when x < p, otherwise zero; the time does not depend on x. pub(crate) fn below(x: &[u64; N], p: &[u64; N]) -> u64 { let mut borrow = false; diff --git a/lib/crypto/src/montgomery/divsteps.rs b/lib/crypto/src/montgomery/divsteps.rs index a11dbc64df6..eb6e9ea85ea 100644 --- a/lib/crypto/src/montgomery/divsteps.rs +++ b/lib/crypto/src/montgomery/divsteps.rs @@ -1,12 +1,18 @@ //! Inversion modulo an odd p by Bernstein and Yang's divsteps ("Fast //! constant-time gcd computation and modular inversion", TCHES 2019), on a -//! fixed schedule: the step count is the paper's bound for p's width, and no +//! fixed schedule: the step count is a proven bound for p's width, and no //! branch or address depends on the operand. //! +//! A p of at most 256 bits takes the variant whose delta starts at 1/2, for +//! which 590 divsteps suffice at that width (Wuille, libsecp256k1's +//! `doc/safegcd_implementation.md`): 10 batches of 59. A wider one takes +//! delta = 1 and the paper's bound, in batches of 62. +//! //! The integers f, g, d and e are signed 62-bit limbs: each limb but the top -//! one lies in [0, 2^62), and the top one carries the sign. A batch of 62 -//! divsteps reads only the low limbs of f and g, and applying its matrix to -//! the whole integers divides by 2^62 as a shift by one limb. +//! one lies in [0, 2^62), and the top one carries the sign. A batch reads +//! only the low limbs of f and g, and its matrix is scaled to 2^62 whatever +//! its step count, so applying it to the whole integers divides by 2^62 as a +//! shift by one limb. use super::Modulus; use crate::ct; @@ -36,46 +42,100 @@ pub(super) fn invert, const N: usize>(x: &[u64; N], scale: &[u64; let p: Limbs = const { split::(&M::P) }; // p^-1 mod 2^62, from N0 = -p^-1 mod 2^64. let inverse = M::N0.wrapping_neg() & MASK; - // Theorem 11.2: divstep^m(1, f, g) leaves g = 0 for m = (49 d + 57) / 17 - // where f^2 + 4 g^2 <= 5 2^(2d) and d >= 46; f = p and g < p meet it - // with d p's width in bits. - let batches = const { - assert!(limbs(N) <= LIMBS && M::P[0] & 1 == 1); - let bits = 64 * N as u64 - M::P[N - 1].leading_zeros() as u64; - assert!(bits >= 46); - ((49 * bits + 57) / 17).div_ceil(62) as usize - }; + let half = const { width::() <= 256 }; + let mut f = p; + let mut g = split::(x); + let mut d = [0; LIMBS]; + let mut e = split::(scale); + // eta = -delta from delta = 1, or zeta = -(delta + 1/2) from delta = 1/2: + // both -1. + let mut eta = -1; + if half { + for _ in 0..10 { + let t; + (eta, t) = divsteps::<59, true>(eta, f[0] as u64, g[0] as u64); + apply(&t, &mut f, &mut g, n); + update(&t, &mut d, &mut e, &p, inverse, n); + } + } else { + for _ in 0..batches::() { + let t; + (eta, t) = divsteps::<62, false>(eta, f[0] as u64, g[0] as u64); + apply(&t, &mut f, &mut g, n); + update(&t, &mut d, &mut e, &p, inverse, n); + } + } + assert!(is_zero(&g, n), "divsteps left g nonzero"); + finish(&f, d, &p, n) +} + +/// x^-1 `scale` mod p for 0 < x < p and scale < p, in variable time: for +/// public x alone. Each batch of 62 divsteps shifts out runs of g's zeros at +/// once and clears up to six of its low bits with one multiple of f, and +/// the batches stop once g is zero (Wuille, libsecp256k1's +/// `doc/safegcd_implementation.md`, section 6). +pub(super) fn invert_public, const N: usize>(x: &[u64; N], scale: &[u64; N]) -> [u64; N] { + let n = limbs(N); + let p: Limbs = const { split::(&M::P) }; + let inverse = M::N0.wrapping_neg() & MASK; let mut f = p; let mut g = split::(x); let mut d = [0; LIMBS]; let mut e = split::(scale); - // eta = -delta, from delta = 1. let mut eta = -1; - for _ in 0..batches { + let mut done = 0; + while !is_zero(&g, n) { + assert!(done < batches::(), "divsteps left g nonzero"); let t; - (eta, t) = divsteps(eta, f[0] as u64, g[0] as u64); + (eta, t) = divsteps_public(eta, f[0] as u64, g[0] as u64); apply(&t, &mut f, &mut g, n); update(&t, &mut d, &mut e, &p, inverse, n); + done += 1; } - // f = gcd(p, x) up to sign, and d f = x^-1 scale mod p. + finish(&f, d, &p, n) +} + +/// Batches of 62 delta = 1 divsteps that suffice for p. Theorem 11.2: +/// divstep^m(1, f, g) leaves g = 0 for m = (49 d + 57) / 17 where +/// f^2 + 4 g^2 <= 5 2^(2d) and d >= 46; f = p and g < p meet it with d p's +/// width in bits. +const fn batches, const N: usize>() -> usize { + let bits = width::(); + assert!(bits >= 46 && limbs(N) <= LIMBS && M::P[0] & 1 == 1); + ((49 * bits + 57) / 17).div_ceil(62) as usize +} + +fn is_zero(g: &Limbs, n: usize) -> bool { let mut rest = 0; for &limb in &g[..n] { rest |= limb; } - assert_eq!(rest, 0, "divsteps left g nonzero"); - // d lies in (-2p, p): add p if it is negative, negate it with f, and add - // p if it is negative again. - add_if_negative(&mut d, &p, n); + rest == 0 +} + +/// x^-1 scale mod p from the final f = gcd(p, x) = +-1 and d, d f = x^-1 +/// scale mod p. d lies in (-2p, p): add p if it is negative, negate it with +/// f, and add p if it is negative again. +fn finish(f: &Limbs, mut d: Limbs, p: &Limbs, n: usize) -> [u64; N] { + add_if_negative(&mut d, p, n); negate_if(&mut d, ct::opaque((f[n - 1] >> 63) as u64), n); - add_if_negative(&mut d, &p, n); + add_if_negative(&mut d, p, n); join(&d) } -/// 62 divsteps on the low limbs of f and g: the new eta and the matrix. +/// p's width in bits. +const fn width, const N: usize>() -> u64 { + 64 * N as u64 - M::P[N - 1].leading_zeros() as u64 +} + +/// STEPS divsteps on the low limbs of f and g: the new eta and the matrix, +/// scaled by 2^(62 - STEPS) to 2^62. With HALF, eta is zeta = -(delta + +/// 1/2), whose step after a swap is -zeta - 2 rather than -eta - 1. #[inline(always)] -fn divsteps(mut eta: i64, mut f: u64, mut g: u64) -> (i64, Matrix) { - let (mut u, mut v, mut q, mut r) = (1u64, 0u64, 0u64, 1u64); - for _ in 0..62 { +fn divsteps(mut eta: i64, mut f: u64, mut g: u64) -> (i64, Matrix) { + let scale = 1u64 << (62 - STEPS); + let (mut u, mut v, mut q, mut r) = (scale, 0u64, 0u64, scale); + for _ in 0..STEPS { // `odd`: g is odd; `swap`: also delta > 0, so the step becomes // (1 - delta, g, (g - f) / 2) rather than (1 + delta, f, (g + f) / 2). let odd = ct::opaque(((g << 63) as i64 >> 63) as u64); @@ -91,11 +151,47 @@ fn divsteps(mut eta: i64, mut f: u64, mut g: u64) -> (i64, Matrix) { u = u.wrapping_add(hq & swap) << 1; v = v.wrapping_add(hr & swap) << 1; (q, r) = (hq, hr); - eta = (eta ^ swap as i64).wrapping_add(!swap as i64); + eta = match HALF { + true => (eta ^ swap as i64).wrapping_sub(1), + false => (eta ^ swap as i64).wrapping_add(!swap as i64), + }; } (eta, Matrix { u: u as i64, v: v as i64, q: q as i64, r: r as i64 }) } +/// 62 delta = 1 divsteps on the low limbs of f and g in variable time, as +/// [`divsteps`] computes them: a run of z zeros of g is z steps that halve g +/// at once; an odd g with delta > 0 swaps as one step would, to (g, -f), +/// after which up to min(1 - delta, 6) steps each add f or not, which one +/// multiple w f with g + w f = 0 mod 2^k decides together. +fn divsteps_public(mut eta: i64, mut f: u64, mut g: u64) -> (i64, Matrix) { + let (mut u, mut v, mut q, mut r) = (1i64, 0i64, 0i64, 1i64); + let mut left = 62; + loop { + let zeros = (g | 1 << left).trailing_zeros(); + g >>= zeros; + u <<= zeros; + v <<= zeros; + eta -= zeros as i64; + left -= zeros; + if left == 0 { + break; + } + if eta < 0 { + (eta, f, g, u, v, q, r) = (-eta, g, f.wrapping_neg(), q, r, -u, -v); + } + let k = (eta + 1).min(left as i64).min(6) as u32; + // f^-1 mod 64: f is its own inverse mod 8, and one Newton step + // doubles that. + let f_inverse = f.wrapping_mul(2u64.wrapping_sub(f.wrapping_mul(f))); + let w = g.wrapping_mul(f_inverse).wrapping_neg() & ((1 << k) - 1); + g = g.wrapping_add(f.wrapping_mul(w)); + q += u * w as i64; + r += v * w as i64; + } + (eta, Matrix { u, v, q, r }) +} + /// a b + c, which the callers' bounds keep within an i128. #[inline(always)] fn mac(a: i64, b: i64, c: i128) -> i128 { @@ -267,6 +363,48 @@ mod tests { } } + /// One divstep by its definition, with delta doubled so that 1/2 is 1: + /// (1 - delta, g, (g - f) / 2) for delta > 0 and g odd, otherwise + /// (1 + delta, f, (g + (g mod 2) f) / 2). + fn step(delta2: i64, f: i128, g: i128) -> (i64, i128, i128) { + if delta2 > 0 && g & 1 == 1 { (2 - delta2, g, (g - f) >> 1) } else { (2 + delta2, f, (g + (g & 1) * f) >> 1) } + } + + /// A batch's matrix and eta for odd f and any g of up to 61 bits and + /// either sign, against `steps` divsteps by the definition: 2^62 (f', g') + /// = (u f + v g, q f + r g), and the new delta. `to_delta2` reads the + /// batch's eta as doubled delta. + fn check_batch(steps: u32, to_delta2: fn(i64) -> i64, batch: fn(i64, u64, u64) -> (i64, Matrix)) { + let mut seed = 0x5eed_d1d5_u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for case in 0..20_000 { + let f = ((next() as i64) >> 3) | 1; + let g = (next() as i64) >> 3; + let eta = (next() % 41) as i64 - 20; + let (eta_out, t) = batch(eta, f as u64, g as u64); + let (mut delta2, mut ff, mut gg) = (to_delta2(eta), f as i128, g as i128); + for _ in 0..steps { + (delta2, ff, gg) = step(delta2, ff, gg); + } + let (f, g) = (f as i128, g as i128); + assert_eq!(t.u as i128 * f + t.v as i128 * g, ff << 62, "case {case}: f row"); + assert_eq!(t.q as i128 * f + t.r as i128 * g, gg << 62, "case {case}: g row"); + assert_eq!(to_delta2(eta_out), delta2, "case {case}: delta"); + } + } + + #[test] + fn batches_are_their_divsteps() { + check_batch(62, |eta| -2 * eta, divsteps::<62, false>); + check_batch(59, |zeta| -2 * zeta - 1, divsteps::<59, true>); + check_batch(62, |eta| -2 * eta, divsteps_public); + } + #[test] fn updates_stay_in_range_and_congruent() { check_update::(); diff --git a/lib/crypto/src/nist.rs b/lib/crypto/src/nist.rs index 99a4cb0c9d8..dcd1f902822 100644 --- a/lib/crypto/src/nist.rs +++ b/lib/crypto/src/nist.rs @@ -14,7 +14,7 @@ use crate::{ Error, - montgomery::{self, Form, Modulus, Residue, below, from_be, to_be}, + montgomery::{self, Computation, Form, Modulus, Residue, below, from_be, to_be}, secret::{Workspace, wipe64}, }; @@ -34,7 +34,7 @@ pub(crate) trait Curve: Sized + 'static { /// G, 3G, ..., 63G, for verification's width-7 recoding of u1. fn odd_multiples_of_g() -> &'static [Affine; 32]; /// scalar G, on a fixed schedule. - fn base_mul(scalar: &Scalar) -> Point { + fn base_mul>(scalar: &Scalar) -> Point { Point::base().mul(scalar) } } @@ -134,6 +134,10 @@ fn magnitude(d: i8) -> (u64, u64) { struct PublicScalar([u64; N]); impl PublicScalar { + /// k^-1 mod n for 0 < k < n, in variable time. + fn inverse>(&self) -> Residue { + Residue::inverse_of_public(&self.0) + } /// The width-w NAF; k < n < 2^(64N) - 2^7, and 385 digits hold N = 6. fn naf(&self, w: u32) -> [i8; 385] { let mut digits = [0i8; 385]; @@ -148,41 +152,41 @@ const fn windows(n: usize) -> usize { (64 * n + 2) / 5 + 1 } -/// An affine point (x, y), a table entry. -pub(crate) struct Affine, const N: usize> { - x: Coord, - y: Coord, +/// An affine point (x, y), a table entry, in the arithmetic of `F`, a +/// modulus type of the curve's p. +pub(crate) struct Affine, const N: usize, F: Modulus = >::Field> { + x: Residue, + y: Residue, + curve: core::marker::PhantomData, } -impl, const N: usize> Clone for Affine { +impl, const N: usize, F: Modulus> Clone for Affine { fn clone(&self) -> Self { *self } } -impl, const N: usize> Copy for Affine {} +impl, const N: usize, F: Modulus> Copy for Affine {} -impl, const N: usize> Affine { +impl, const N: usize, F: Modulus> Affine { + fn new(x: Residue, y: Residue) -> Self { + Self { x, y, curve: core::marker::PhantomData } + } + /// A table entry in `F`'s arithmetic. + fn from_table(entry: &Affine) -> Self { + Self::new(entry.x.cast(), entry.y.cast()) + } /// row[|d| - 1], with y negated by mask when d < 0, reading every entry. /// For d = 0 it is row[0], which the caller discards by mask: no entry /// is the identity, so no arithmetic runs on it. - fn lookup(row: &[Self; 16], d: i8) -> Self { + fn lookup(row: &[Affine; 16], d: i8) -> Self { let (magnitude, negative) = magnitude(d); - let mut chosen = row[0]; - // Indexing, not an iterator adaptor, which the shipped profile keeps - // out of line. - for j in 1..16 { - let take = 1 ^ crate::ct::nonzero(magnitude ^ (j as u64 + 1)); - chosen.x = Residue::select(&chosen.x, &row[j].x, take); - chosen.y = Residue::select(&chosen.y, &row[j].y, take); - } + let [x, y] = F::select(row, |e| [e.x.words(), e.y.words()], magnitude.wrapping_sub(1)); + let entry = Affine::::new(Residue::from_montgomery(x), Residue::from_montgomery(y)); + let mut chosen = Self::from_table(&entry); chosen.y = Residue::select(&chosen.y, &Residue::zero().sub(&chosen.y), negative); chosen } - fn point(&self) -> Point { - Point { - x: self.x, - y: self.y, - z: Residue::one(), - } + fn point(&self) -> Point { + Point::new(self.x, self.y, Residue::one()) } } @@ -240,10 +244,7 @@ impl, const N: usize> Build { i += 1; } let mut inverse = prefix[K - 1].inverted(); - let mut out = [Affine { - x: Residue::zero(), - y: Residue::zero(), - }; K]; + let mut out = [Affine { x: Residue::zero(), y: Residue::zero(), curve: core::marker::PhantomData }; K]; let mut i = K; while i > 0 { i -= 1; @@ -253,6 +254,7 @@ impl, const N: usize> Build { out[i] = Affine { x: points[i][0].times(&z2), y: points[i][1].times(&z2.times(&z_inverse)), + curve: core::marker::PhantomData, }; } out @@ -298,10 +300,7 @@ impl, const N: usize> Build { i += 1; } let mut inverse = running.inverted(); - let mut out = [[Affine { - x: Residue::zero(), - y: Residue::zero(), - }; 16]; R]; + let mut out = [[Affine { x: Residue::zero(), y: Residue::zero(), curve: core::marker::PhantomData }; 16]; R]; let mut i = 16 * R; while i > 0 { i -= 1; @@ -312,6 +311,7 @@ impl, const N: usize> Build { out[i / 16][i % 16] = Affine { x: p[0].times(&z2), y: p[1].times(&z2.times(&z_inverse)), + curve: core::marker::PhantomData, }; } out @@ -326,44 +326,45 @@ static P256_G_ODD: [Affine; 32] = Build::::odd_multiples(); static P384_G_ODD: [Affine; 32] = Build::::odd_multiples(); /// The odd multiples P, 3P, ..., 15P of a public point, for verification. -type OddMultiples = [Point; 8]; +type OddMultiples = [Point; 8]; -/// Jacobian coordinates (X : Y : Z) for x = X/Z^2, y = Y/Z^3; Z = 0 is the -/// point at infinity. -pub(crate) struct Point, const N: usize> { - x: Coord, - y: Coord, - z: Coord, +/// Jacobian coordinates (X : Y : Z) for x = X/Z^2, y = Y/Z^3, in the +/// arithmetic of `F`, a modulus type of the curve's p; Z = 0 is the point at +/// infinity. +pub(crate) struct Point, const N: usize, F: Modulus = >::Field> { + x: Residue, + y: Residue, + z: Residue, + curve: core::marker::PhantomData, } -impl, const N: usize> Clone for Point { +impl, const N: usize, F: Modulus> Clone for Point { fn clone(&self) -> Self { *self } } -impl, const N: usize> Copy for Point {} +impl, const N: usize, F: Modulus> Copy for Point {} // SAFETY: three residues, and zero is a valid point representation. -unsafe impl, const N: usize> crate::secret::Zeroable for Point {} -impl, const N: usize> Point { +unsafe impl, const N: usize, F: Modulus> crate::secret::Zeroable for Point {} +impl, const N: usize, F: Modulus> Point { + fn new(x: Residue, y: Residue, z: Residue) -> Self { + Self { x, y, z, curve: core::marker::PhantomData } + } fn identity() -> Self { - Self { - x: Residue::zero(), - y: Residue::one(), - z: Residue::zero(), - } + Self::new(Residue::zero(), Residue::one(), Residue::zero()) } pub(crate) fn base() -> Self { - Self { - x: Residue::from_montgomery(C::GX), - y: Residue::from_montgomery(C::GY), - z: Residue::one(), - } + Self::new(Residue::from_montgomery(C::GX), Residue::from_montgomery(C::GY), Residue::one()) + } + /// The same point in another modulus type's arithmetic. + fn cast>(&self) -> Point { + Point::new(self.x.cast(), self.y.cast(), self.z.cast()) } fn select(a: &Self, b: &Self, bit: u64) -> Self { - Self { - x: Residue::select(&a.x, &b.x, bit), - y: Residue::select(&a.y, &b.y, bit), - z: Residue::select(&a.z, &b.z, bit), - } + Self::new( + Residue::select(&a.x, &b.x, bit), + Residue::select(&a.y, &b.y, bit), + Residue::select(&a.z, &b.z, bit), + ) } /// "dbl-2001-b" for a = -3 (Bernstein and Lange, Explicit-Formulas /// Database), 3M + 5S. Infinity doubles to Z = 0, and no point of a @@ -379,7 +380,7 @@ impl, const N: usize> Point { let z = self.y.add(&self.z).square().sub(&gamma).sub(&delta); let gamma2 = gamma.square().double(); let y = alpha.mul(&beta4.sub(&x)).sub(&gamma2.double().double()); - Self { x, y, z } + Self::new(x, y, z) } /// scalar * self on a fixed schedule, for a point of prime order n: /// Booth's signed 5-bit windows of the odd representative k' over the @@ -399,11 +400,7 @@ impl, const N: usize> Point { /// That last addition can double, and does so by mask; it cannot cancel, /// since k' is not 0 mod n. pub(crate) fn mul(&self, scalar: &Scalar) -> Self { - let mut table = [*self; 16]; - table[1] = self.double(); - for j in 2..16 { - table[j] = table[j - 1].add_public(self); - } + let table = self.multiples(); let k = scalar.representative(); let top = windows(N) - 1; let mut r = Self::lookup(&table, booth(&k.0, top)); @@ -420,6 +417,18 @@ impl, const N: usize> Point { } r } + /// table[j] = (j + 1) self for the public self: each even multiple a + /// doubling, each odd one the even one below it plus self. + fn multiples(&self) -> [Self; 16] { + let mut table = [*self; 16]; + for j in (1..16).step_by(2) { + table[j] = table[j / 2].double(); + if j + 1 < 16 { + table[j + 1] = table[j].add_public(self); + } + } + table + } /// self + q for points that are not the identity, 12M + 4S, and one when /// they are equal (h = r = 0), where the formula fails. fn add_incomplete(&self, q: &Self) -> (Self, u64) { @@ -437,24 +446,22 @@ impl, const N: usize> Point { let x = r.square().sub(&h3).sub(&u.double()); let y = r.mul(&u.sub(&x)).sub(&s1.mul(&h3)); let z = h.mul(&self.z.mul(&q.z)); - (Self { x, y, z }, h.is_zero() & r.is_zero()) + (Self::new(x, y, z), h.is_zero() & r.is_zero()) } /// table[|d| - 1] for table[j] = (j + 1) P, with Y negated by mask for /// d < 0, reading every entry. For d = 0 it is table[0], which the caller /// discards by mask. fn lookup(table: &[Self; 16], d: i8) -> Self { let (magnitude, negative) = magnitude(d); - let mut chosen = table[0]; - for j in 1..16 { - chosen = Self::select(&chosen, &table[j], 1 ^ crate::ct::nonzero(magnitude ^ (j as u64 + 1))); - } + let [x, y, z] = F::select(table, |e| [e.x.words(), e.y.words(), e.z.words()], magnitude.wrapping_sub(1)); + let mut chosen = Self::new(Residue::from_montgomery(x), Residue::from_montgomery(y), Residue::from_montgomery(z)); let negated = Residue::zero().sub(&chosen.y); chosen.y = Residue::select(&chosen.y, &negated, negative); chosen } /// self + q for an affine q, 8M + 3S, and one when the operands are /// equal (h = r = 0), where the formula fails. self is not the identity. - fn add_affine(&self, q: &Affine) -> (Self, u64) { + fn add_affine(&self, q: &Affine) -> (Self, u64) { let z1 = self.z.square(); let u2 = q.x.mul(&z1); let s2 = q.y.mul(&self.z.mul(&z1)); @@ -466,7 +473,7 @@ impl, const N: usize> Point { let x = r.square().sub(&h3).sub(&u.double()); let y = r.mul(&u.sub(&x)).sub(&self.y.mul(&h3)); let z = h.mul(&self.z); - (Self { x, y, z }, h.is_zero() & r.is_zero()) + (Self::new(x, y, z), h.is_zero() & r.is_zero()) } /// scalar G by a comb, with no doublings: the odd representative's Booth /// digit i selects from row i, j 2^(5i) G, and one mixed addition adds @@ -484,7 +491,7 @@ impl, const N: usize> Point { fn comb_mul(table: &[[Affine; 16]; R], scalar: &Scalar) -> Self { let k = scalar.representative(); // k' is odd, so digit 0 is nonzero. - let mut r = Affine::lookup(&table[0], booth(&k.0, 0)).point(); + let mut r = Affine::::lookup(&table[0], booth(&k.0, 0)).point(); for i in 1..R { let d = booth(&k.0, i); let (mut sum, equal) = r.add_affine(&Affine::lookup(&table[i], d)); @@ -496,7 +503,7 @@ impl, const N: usize> Point { r } /// self + q for a public affine q, branching on the values. - fn add_affine_public(&self, q: &Affine) -> Self { + fn add_affine_public(&self, q: &Affine) -> Self { if self.z.is_zero() == 1 { return q.point(); } @@ -522,13 +529,9 @@ impl, const N: usize> Point { sum } fn neg(&self) -> Self { - Self { - x: self.x, - y: Residue::zero().sub(&self.y), - z: self.z, - } + Self::new(self.x, Residue::zero().sub(&self.y), self.z) } - fn odd_multiples(&self) -> OddMultiples { + fn odd_multiples(&self) -> OddMultiples { let twice = self.double(); let mut odd = [*self; 8]; for j in 1..8 { @@ -549,13 +552,10 @@ impl, const N: usize> Point { r = r.double(); let d = d1[i]; if d > 0 { - r = r.add_affine_public(&g[d as usize / 2]); + r = r.add_affine_public(&Affine::from_table(&g[d as usize / 2])); } else if d < 0 { - let entry = g[(-d) as usize / 2]; - r = r.add_affine_public(&Affine { - x: entry.x, - y: Residue::zero().sub(&entry.y), - }); + let entry = Affine::::from_table(&g[(-d) as usize / 2]); + r = r.add_affine_public(&Affine::new(entry.x, Residue::zero().sub(&entry.y))); } let d = d2[i]; if d > 0 { @@ -578,7 +578,7 @@ impl, const N: usize> Point { Ok(()) } /// The uncompressed SEC 1 encoding, 1 + 16N bytes. - pub(crate) fn encode(&self, out: &mut [u8]) -> Result<(), Error> { + fn encode(&self, out: &mut [u8]) -> Result<(), Error> { if out.len() != 1 + N * 16 { return Err(Error::Length); } @@ -589,6 +589,88 @@ impl, const N: usize> Point { to_be(&y, &mut out[1 + N * 8..]); Ok(()) } + /// Whether this verification result's x is r mod n: x(R) = X / Z^2 is + /// r mod n exactly when X = r' Z^2 for r' = r, or r' = r + n where that + /// is still below p: no inversion. + fn x_is(&self, r: &[u64; N]) -> bool { + if self.z.is_zero() == 1 { + return false; + } + let z2 = self.z.square(); + let r_field = Residue::::from_integer(r).expect("r < n < p"); + if self.x.eq(&r_field.mul(&z2)) == 1 { + return true; + } + let mut r_plus_n = [0; N]; + let mut carry = false; + for i in 0..N { + let (sum, c1) = r[i].overflowing_add(C::Order::P[i]); + let (sum, c2) = sum.overflowing_add(u64::from(carry)); + r_plus_n[i] = sum; + carry = c1 | c2; + } + match (carry, Residue::::from_integer(&r_plus_n)) { + (false, Some(r_field)) => self.x.eq(&r_field.mul(&z2)) == 1, + _ => false, + } + } +} + +/// The dispatched computations: each runs in the arithmetic +/// `Modulus::dispatch` chooses for the curve's field, its points cast into +/// it, so no field operation inside checks the CPU. +struct Agree<'a, C: Curve, const N: usize> { + peer: &'a Point, + scalar: &'a Scalar, + output: &'a mut [u8], +} +impl, const N: usize> Computation for Agree<'_, C, N> { + type Output = (); + fn run>(self) { + let product = Workspace(self.peer.cast::().mul(self.scalar)); + let mut coordinates = Workspace([[0; N]; 2]); + let [x, y] = &mut coordinates.0; + // Both curves have prime order and cofactor one; validated nonzero + // scalars and finite peers cannot multiply to infinity. + product.0.affine(x, y).expect("validated prime-order point and nonzero scalar"); + to_be(x, self.output); + } +} + +struct PublicKey<'a, C: Curve, const N: usize> { + scalar: &'a Scalar, + encoded: &'a mut [u8], +} +impl, const N: usize> Computation for PublicKey<'_, C, N> { + type Output = Point; + fn run>(self) -> Point { + let point = C::base_mul::(self.scalar); + // The base point's multiple by a nonzero scalar below n is finite. + point.encode(self.encoded).expect("nonzero prime-order scalar"); + point.cast() + } +} + +struct Verify<'a, C: Curve, const N: usize> { + key: &'a Point, + u1: PublicScalar, + u2: PublicScalar, + r: [u64; N], +} +impl, const N: usize> Computation for Verify<'_, C, N> { + type Output = bool; + fn run>(self) -> bool { + self.key.cast::().joint(&self.u1, &self.u2).x_is(&self.r) + } +} + +impl, const N: usize> Point { + /// scalar G and its uncompressed SEC 1 encoding into `encoded`, 1 + 16N + /// bytes, for a valid secret scalar. + pub(crate) fn public_key(scalar: &Scalar, encoded: &mut [u8]) -> Self { + assert_eq!(encoded.len(), 1 + N * 16); + C::Field::dispatch(PublicKey { scalar, encoded }) + } /// An uncompressed SEC 1 point on the curve. pub(crate) fn decode(encoded: &[u8]) -> Result { if encoded.len() != 1 + N * 16 { @@ -610,26 +692,13 @@ impl, const N: usize> Point { if y.square().eq(&rhs) != 1 { return Err(Error::InvalidPoint); } - Ok(Point { - x, - y, - z: Residue::one(), - }) + Ok(Point::new(x, y, Residue::one())) } /// The x-coordinate of scalar * self into `output`, 8N bytes. The product - /// and its affine coordinates are the shared secret's, erased here once + /// and its affine coordinates are the shared secret's, erased once /// `output` is written. pub(crate) fn agree(&self, scalar: &Scalar, output: &mut [u8]) { - let product = Workspace(self.mul(scalar)); - let mut coordinates = Workspace([[0; N]; 2]); - let [x, y] = &mut coordinates.0; - // Both curves have prime order and cofactor one; validated nonzero - // scalars and finite peers cannot multiply to infinity. - product - .0 - .affine(x, y) - .expect("validated prime-order point and nonzero scalar"); - to_be(x, output); + C::Field::dispatch(Agree { peer: self, scalar, output }) } /// ECDSA verification of a DER signature over an 8N-byte digest. All /// inputs are public. @@ -641,31 +710,12 @@ impl, const N: usize> Point { let r = Scalar::::from_bytes(&r[..N * 8]).map_err(|_| Error::InvalidSignature)?; let s = Scalar::::from_bytes(&s[..N * 8]).map_err(|_| Error::InvalidSignature)?; let z = ModN::::from_wide(&from_be(digest)); - let inverse = ModN::::from_wide(&s.0).invert(); + let inverse = PublicScalar(s.0).inverse::(); let u1 = PublicScalar(z.mul(&inverse).integer()); let u2 = PublicScalar(ModN::::from_wide(&r.0).mul(&inverse).integer()); - let result = self.joint(&u1, &u2); - if result.z.is_zero() == 1 { - return Err(Error::InvalidSignature); - } - // x(R) = X / Z^2 is r mod n exactly when X = r' Z^2 for r' = r, or - // r' = r + n where that is still below p: no inversion. - let z2 = result.z.square(); - let r_field = Coord::::from_integer(&r.0).expect("r < n < p"); - if result.x.eq(&r_field.mul(&z2)) == 1 { - return Ok(()); - } - let mut r_plus_n = [0; N]; - let mut carry = false; - for i in 0..N { - let (sum, c1) = r.0[i].overflowing_add(C::Order::P[i]); - let (sum, c2) = sum.overflowing_add(u64::from(carry)); - r_plus_n[i] = sum; - carry = c1 | c2; - } - match (carry, Coord::::from_integer(&r_plus_n)) { - (false, Some(r_field)) if result.x.eq(&r_field.mul(&z2)) == 1 => Ok(()), - _ => Err(Error::InvalidSignature), + match C::Field::dispatch(Verify { key: self, u1, u2, r: r.0 }) { + true => Ok(()), + false => Err(Error::InvalidSignature), } } } @@ -715,11 +765,32 @@ macro_rules! modulus { } }; } -/// P-256's prime, with its special reduction step. +/// P-256's prime, with its special reduction step, its arithmetic word by +/// word, and kernels where the CPU has them (`arch::p256`). pub(crate) struct P256Field; impl Modulus<4> for P256Field { const P: [u64; 4] = [0xffffffffffffffff, 0x00000000ffffffff, 0, 0xffffffff00000001]; const FORM: Form = Form::P256; + #[inline(always)] + fn add(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + montgomery::p256::add(a, b) + } + #[inline(always)] + fn sub(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + montgomery::p256::sub(a, b) + } + #[inline(always)] + fn mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] { + montgomery::p256::mul(a, b) + } + #[inline(always)] + fn square(a: &[u64; 4]) -> [u64; 4] { + montgomery::p256::square(a) + } + #[inline(always)] + fn dispatch>(computation: C) -> C::Output { + crate::arch::p256(computation) + } } modulus!(P256Order, 4, [0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000]); modulus!( @@ -757,7 +828,7 @@ impl Curve<4> for P256 { fn odd_multiples_of_g() -> &'static [Affine; 32] { &P256_G_ODD } - fn base_mul(scalar: &Scalar) -> Point { + fn base_mul>(scalar: &Scalar) -> Point { Point::comb_mul(&P256_COMB, scalar) } const B: [u64; 4] = montgomery::encode( @@ -919,12 +990,12 @@ mod tests { /// The fixed-schedule Booth multiplication equals the public joint /// multiplication, a different recoding and addition, for scalars below /// n including those with extreme digits. - fn check_mul, const N: usize>() { + fn check_mul, const N: usize, F: Modulus>() { let zero = PublicScalar([0; N]); for k in scalars(&C::Order::P) { let k = ModN::::from_wide(&k).integer(); - let ours = Point::::base().mul(&Scalar(k, core::marker::PhantomData)); - let theirs = Point::::identity().joint(&PublicScalar(k), &zero); + let ours = Point::::base().mul(&Scalar(k, core::marker::PhantomData)); + let theirs = Point::::identity().joint(&PublicScalar(k), &zero); let (mut a, mut b) = ([0; N], [0; N]); let (mut c, mut d) = ([0; N], [0; N]); match (ours.affine(&mut a, &mut b), theirs.affine(&mut c, &mut d)) { @@ -937,12 +1008,12 @@ mod tests { /// The public path's exceptional sums: equal summands double, and /// opposite ones cancel to the identity. - fn check_exceptional_sums, const N: usize>() { + fn check_exceptional_sums, const N: usize, F: Modulus>() { let mut one = [0; N]; one[0] = 1; let (mut a, mut b) = ([0; N], [0; N]); let (mut c, mut d) = ([0; N], [0; N]); - let g = Point::::base(); + let g = Point::::base(); g.joint(&PublicScalar(one), &PublicScalar(one)).affine(&mut a, &mut b).unwrap(); g.double().affine(&mut c, &mut d).unwrap(); assert_eq!((a, b), (c, d)); @@ -951,11 +1022,11 @@ mod tests { #[test] fn public_sums_of_equal_and_opposite_points() { - check_exceptional_sums::(); - check_exceptional_sums::(); + check_exceptional_sums::(); + check_exceptional_sums::(); } - fn affine_of, const N: usize>(p: &Point) -> ([u64; N], [u64; N]) { + fn affine_of, const N: usize, F: Modulus>(p: &Point) -> ([u64; N], [u64; N]) { let (mut x, mut y) = ([0; N], [0; N]); p.affine(&mut x, &mut y).unwrap(); (x, y) @@ -997,6 +1068,10 @@ mod tests { /// addition doubles. #[test] fn the_comb_matches_the_public_path() { + check_comb::(); + } + + fn check_comb>() { use num_bigint::BigInt; let n = integer(&P256Order::P); let mut ks: std::vec::Vec = @@ -1012,7 +1087,7 @@ mod tests { b[..chunk.len()].copy_from_slice(chunk); words[i] = u64::from_le_bytes(b); } - let comb = P256::base_mul(&Scalar(words, core::marker::PhantomData)); + let comb = P256::base_mul::(&Scalar(words, core::marker::PhantomData)); assert_eq!(affine_of(&comb), reference::(&k), "k = {k:x}"); } } @@ -1020,7 +1095,7 @@ mod tests { /// The scalars k = 2d mod n whose representative's first digit is d, /// for which the last addition of [`Point::mul`] doubles: at least one /// exists per curve, and each multiplies as the public path does. - fn check_last_addition_doubles, const N: usize>() { + fn check_last_addition_doubles, const N: usize, F: Modulus>() { use num_bigint::BigInt; let n = integer(&C::Order::P); let mut found = 0; @@ -1038,20 +1113,38 @@ mod tests { continue; } found += 1; - assert_eq!(affine_of(&Point::::base().mul(&scalar)), reference::(&k), "d = {d}"); + assert_eq!(affine_of(&Point::::base().mul(&scalar)), reference::(&k), "d = {d}"); } assert!(found > 0); } #[test] fn the_last_addition_doubles_where_it_must() { - check_last_addition_doubles::(); - check_last_addition_doubles::(); + check_last_addition_doubles::(); + check_last_addition_doubles::(); } #[test] fn booth_multiplication_matches_the_public_path() { - check_mul::(); - check_mul::(); + check_mul::(); + check_mul::(); + } + + /// The point checks above in the arithmetic P-256's dispatch chooses: + /// the kernels, on a CPU that has them. + #[test] + fn p256_points_in_the_dispatched_arithmetic() { + struct Checks; + impl Computation<4> for Checks { + type Output = &'static str; + fn run>(self) -> &'static str { + check_mul::(); + check_exceptional_sums::(); + check_last_addition_doubles::(); + check_comb::(); + core::any::type_name::() + } + } + std::println!("p-256 points in {}", P256Field::dispatch(Checks)); } } diff --git a/lib/crypto/src/poly1305.rs b/lib/crypto/src/poly1305.rs index 79ed112233b..471cf57ab75 100644 --- a/lib/crypto/src/poly1305.rs +++ b/lib/crypto/src/poly1305.rs @@ -12,6 +12,7 @@ use crate::secret::{Secret, erase}; /// RFC 8439 §2.5's tag of `message` under the one-time `key`. pub fn tag(key: &[u8; 32], message: &[u8]) -> [u8; 16] { + let _protection = crate::timing::protect(); let mut mac = Poly1305::new(key); mac.update(message); mac.finish() @@ -27,6 +28,7 @@ pub fn verify(key: &[u8; 32], message: &[u8], expected: &[u8; 16]) -> Result<(), /// to 16. pub fn verify_truncated(key: &[u8; 32], message: &[u8], expected: &[u8; N]) -> Result<(), Error> { const { assert!(N >= 1 && N <= 16) }; + let _protection = crate::timing::protect(); let tag = Secret(tag(key, message)); if crate::ct::equal(&tag.0[..N], expected) { Ok(()) } else { Err(Error::Authentication) } } diff --git a/lib/crypto/src/rustls_provider/hash.rs b/lib/crypto/src/rustls_provider/hash.rs index 0c90f5c7fee..ce0161fbd38 100644 --- a/lib/crypto/src/rustls_provider/hash.rs +++ b/lib/crypto/src/rustls_provider/hash.rs @@ -71,6 +71,7 @@ impl hmac::Hmac for Algorithm { } impl hmac::Key for Mac { fn sign_concat(&self, first: &[u8], middle: &[&[u8]], last: &[u8]) -> hmac::Tag { + let _protection = crate::timing::protect(); let mut mac = self.0.fork(); for part in core::iter::once(first) .chain(middle.iter().copied()) diff --git a/lib/crypto/src/scrypt.rs b/lib/crypto/src/scrypt.rs index 85e4ea63585..76aa21e8a70 100644 --- a/lib/crypto/src/scrypt.rs +++ b/lib/crypto/src/scrypt.rs @@ -76,6 +76,7 @@ impl Params { /// (2^32 - 1) * 32 bytes. `workspace` is exactly /// [`Params::workspace_len`] words. pub fn derive(passphrase: &[u8], salt: &[u8], params: Params, workspace: &mut [u32], output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); let workspace = Erased(workspace); if workspace.0.len() != params.workspace_len() || output.is_empty() || output.len().div_ceil(32) > u32::MAX as usize { return Err(Error::Length); diff --git a/lib/crypto/src/siphash.rs b/lib/crypto/src/siphash.rs index 9d7fce3f476..7603b0274ec 100644 --- a/lib/crypto/src/siphash.rs +++ b/lib/crypto/src/siphash.rs @@ -24,6 +24,7 @@ fn compress(v: &mut [u64; 4], m: u64) { } pub fn siphash24(key: &Key, data: &[u8]) -> u64 { + let _protection = crate::timing::protect(); let (k0, k1) = key.split_at(8); let k0 = k0.iter().rev().fold(0u64, |a, &b| a.rotate_left(8) | u64::from(b)); let k1 = k1.iter().rev().fold(0u64, |a, &b| a.rotate_left(8) | u64::from(b)); diff --git a/lib/crypto/src/ssh.rs b/lib/crypto/src/ssh.rs index 6c7fa78112d..b0bf431d0b3 100644 --- a/lib/crypto/src/ssh.rs +++ b/lib/crypto/src/ssh.rs @@ -81,6 +81,7 @@ impl SealingKey { header: &mut [u8; 4], payload: &mut [u8], ) -> Result<(u32, [u8; 16]), Error> { + let _protection = crate::timing::protect(); if payload.len() > u32::MAX as usize { return Err(Error::Length); } @@ -115,6 +116,7 @@ impl OpeningKey { /// Unauthenticated length is only a framing hint. The caller must bound /// allocations and authenticate the complete packet before using plaintext. pub fn packet_length(&self, seq: u32, encrypted: [u8; 4]) -> u32 { + let _protection = crate::timing::protect(); let mut header = encrypted; self.keys.crypt_header(seq, &mut header); u32::from_be_bytes(header) @@ -126,6 +128,7 @@ impl OpeningKey { payload: &mut [u8], tag: &[u8], ) -> Result<(), Error> { + let _protection = crate::timing::protect(); if tag.len() != 16 || payload.len() > u32::MAX as usize { return Err(Error::Length); } diff --git a/lib/crypto/src/test_arithmetic.rs b/lib/crypto/src/test_arithmetic.rs index 19c2853bd8e..2d3b4bda47b 100644 --- a/lib/crypto/src/test_arithmetic.rs +++ b/lib/crypto/src/test_arithmetic.rs @@ -7,7 +7,7 @@ use crate::{ field25519::{Multiply, Tight}, - montgomery::{Modulus, Residue}, + montgomery::{Computation, Modulus, Residue}, nist::{P256Field, P256Order, P384Field, P384Order}, scalar25519::S, }; @@ -350,9 +350,17 @@ fn field25519_loose_operands_match_integers() { #[test] fn nist_fields_and_orders_match_integers() { + /// The check in the arithmetic the CPU's dispatch chooses, named. + struct Dispatched; + impl Computation<4> for Dispatched { + type Output = &'static str; + fn run>(self) -> &'static str { + check(&Montgomery::(PhantomData), 1); + core::any::type_name::() + } + } // Which P-256 field path this run checks, for a reader of `--nocapture`. - let kernel = crate::arch::p256_mul(&[0; 4], &[0; 4]).is_some() && crate::arch::p256_square(&[0; 4]).is_some(); - std::println!("p-256 field path: {}", if kernel { "the BMI2/ADX kernels" } else { "portable" }); + std::println!("p-256 field path: {}", P256Field::dispatch(Dispatched)); check(&Montgomery::(PhantomData), 1); check(&Montgomery::(PhantomData), 2); check(&Montgomery::(PhantomData), 3); @@ -367,6 +375,26 @@ fn compile_time_montgomery_forms_match_integers() { check(&MontgomeryConst::(PhantomData), 8); } +/// The variable-time inverse of public integers, from 1 to p - 1: the edges +/// and seeded values. +#[test] +fn public_inverses_match_integers() { + fn run, const N: usize>(seed: u64) { + let p = integer(&M::P); + let mut rng = XorShift(seed); + let mut values = boundaries(&p, &p); + values.extend((0..CASES).map(|_| rng.below(&p))); + for a in values.iter().filter(|a| **a != BigUint::from(0u8)) { + let inverse = Residue::::inverse_of_public(&words(a)); + assert_eq!(integer(&inverse.integer()), a.modinv(&p).unwrap(), "1 / {a}"); + } + } + run::(9); + run::(10); + run::(11); + run::(12); +} + /// The compile-time Montgomery constants, and the canonical decoding that /// refuses every value from the modulus up. #[test] diff --git a/lib/crypto/src/timing.rs b/lib/crypto/src/timing.rs new file mode 100644 index 00000000000..4f548ce5ec1 --- /dev/null +++ b/lib/crypto/src/timing.rs @@ -0,0 +1,31 @@ +//! The calling thread's timing protection on ToyOS (`toyos::timing`); off +//! ToyOS, [`protect`] is nothing. +//! +//! The boundary is the operation that is secret by construction: every +//! operation that computes on a key or other secret takes a protection on its +//! first line, which ends with it and nests in any its caller holds. Ciphers, +//! MACs, KDFs, keyed SipHash, [`crate::ct::equal`] and every secret-key and +//! agreement operation do. A plain hash serves public data as often as secret +//! and takes none, and verification holds no secret: **a caller hashing a +//! secret takes [`protect`] around it itself.** + +#[cfg(target_os = "toyos")] +pub use toyos::timing::{protect, Protection}; + +#[cfg(not(target_os = "toyos"))] +#[must_use = "the protection ends when this drops"] +pub struct Protection; + +#[cfg(not(target_os = "toyos"))] +impl Protection { + /// Never: there is no mode off ToyOS. + pub fn is_active(&self) -> bool { + false + } +} + +#[cfg(not(target_os = "toyos"))] +#[inline(always)] +pub fn protect() -> Protection { + Protection +} diff --git a/lib/crypto/src/wpa2.rs b/lib/crypto/src/wpa2.rs index dec7bbdf261..07e1405ad44 100644 --- a/lib/crypto/src/wpa2.rs +++ b/lib/crypto/src/wpa2.rs @@ -133,6 +133,7 @@ impl PairwiseMasterKey { } /// IEEE 802.11 passphrase form; a raw hexadecimal PSK belongs to the caller. pub fn derive_pmk(passphrase: &[u8], ssid: &[u8]) -> Result { + let _protection = crate::timing::protect(); if !(8..=63).contains(&passphrase.len()) || ssid.is_empty() || ssid.len() > 32 { return Err(Error::Length); } @@ -152,10 +153,12 @@ pub fn derive_pmk(passphrase: &[u8], ssid: &[u8]) -> Result Result<[u8; 20], Error> { + let _protection = crate::timing::protect(); Ok(mac(key, parts)?.0) } /// 802.11i PRF: HMAC-SHA1(key, label || 0 || data || counter), counter from 0. pub fn prf(key: &[u8], label: &[u8], data: &[u8], output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); if output.len() > 256 * 20 { return Err(Error::Length); } @@ -173,11 +176,13 @@ pub fn prf(key: &[u8], label: &[u8], data: &[u8], output: &mut [u8]) -> Result<( pub struct KeyEncryptionKey(Aes); impl KeyEncryptionKey { pub fn new(key: [u8; 16]) -> Self { + let _protection = crate::timing::protect(); let key = Secret(key); Self(Aes::aes128(&key.0)) } /// RFC 3394 default-IV unwrap. Output has exactly ciphertext.len()-8 bytes. pub fn unwrap(&self, ciphertext: &[u8], output: &mut [u8]) -> Result<(), Error> { + let _protection = crate::timing::protect(); if ciphertext.len() < 24 || !ciphertext.len().is_multiple_of(8) || output.len() != ciphertext.len() - 8 diff --git a/lib/crypto/src/x25519.rs b/lib/crypto/src/x25519.rs index 542830feeb1..b6885073c6e 100644 --- a/lib/crypto/src/x25519.rs +++ b/lib/crypto/src/x25519.rs @@ -31,9 +31,11 @@ impl SecretKey { Ok(Self::from_bytes(b.0)) } pub fn public_key(&self) -> PublicKey { + let _protection = crate::timing::protect(); PublicKey(crate::ed25519::x25519_public(&self.0.0)) } pub fn agree(self, peer: &PublicKey) -> Result { + let _protection = crate::timing::protect(); let mut result = Secret([0; 32]); multiply(&self.0.0, &peer.0, &mut result.0); if ct::equal(&result.0, &[0; 32]) { diff --git a/lib/crypto/tests/differential.rs b/lib/crypto/tests/differential.rs index a3fbc59bd5a..b7c80aab7ad 100644 --- a/lib/crypto/tests/differential.rs +++ b/lib/crypto/tests/differential.rs @@ -550,7 +550,6 @@ macro_rules! nist_case { } #[test] -#[cfg_attr(not(feature = "portable"), ignore = "the same code in every form: run in the portable one")] fn seeded_p256_against_rustcrypto_and_ring() { support::seeded(CASES, 0x9256, |case, rng| nist_case!(p256, 32, ECDSA_P256_SHA256_ASN1, case, rng)); } diff --git a/lib/crypto/tests/fuzz.rs b/lib/crypto/tests/fuzz.rs index 4346f7760d5..f43a4301b23 100644 --- a/lib/crypto/tests/fuzz.rs +++ b/lib/crypto/tests/fuzz.rs @@ -228,7 +228,6 @@ macro_rules! der_case { } #[test] -#[cfg_attr(not(feature = "portable"), ignore = "the same code in every form: run in the portable one")] fn der_signatures() { seeded(CASES, 0xde5, |case, rng| { der_case!(p256, 32, ECDSA_P256_SHA256_ASN1, b"ffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551", case, rng) diff --git a/lib/crypto/tests/nist.rs b/lib/crypto/tests/nist.rs index 8f0975abd36..d7ae0b889cf 100644 --- a/lib/crypto/tests/nist.rs +++ b/lib/crypto/tests/nist.rs @@ -4,8 +4,8 @@ //! offer is counted outside the interface by the reason it names, and each //! file's cases are held to a count taken from its text apart from the //! parse: its `Count = `, `Len = `, `Msg = ` or `Qx = ` lines, or its -//! `"tcId"`s. ECDSA and ML-KEM, which have no instruction path, run in the -//! `portable` form alone. +//! `"tcId"`s. ML-KEM, which has no instruction path, runs in the `portable` +//! form alone. mod support; @@ -179,7 +179,6 @@ fn cavp_gcm() { /// FIPS 186-4's signature and public-key validation vectors, on P-256 and /// P-384 with SHA-256, SHA-384 and SHA-512, the digest as SEC 1's bits2int. #[test] -#[cfg_attr(not(feature = "portable"), ignore = "the same code in every form: run in the portable one")] fn cavp_ecdsa() { let curve = |r: &Record| -> Option { match r.headers[0].split(',').next().unwrap() { diff --git a/lib/crypto/tests/wycheproof.rs b/lib/crypto/tests/wycheproof.rs index 79a2a9a0787..85c0f42a26a 100644 --- a/lib/crypto/tests/wycheproof.rs +++ b/lib/crypto/tests/wycheproof.rs @@ -193,7 +193,6 @@ fn ed25519() { } #[test] -#[cfg_attr(not(feature = "portable"), ignore = "the same code in every form: run in the portable one")] fn ecdsa_der() { for file in [ "ecdsa_secp256r1_sha256_test.json", @@ -223,7 +222,6 @@ fn ecdsa_der() { } #[test] -#[cfg_attr(not(feature = "portable"), ignore = "the same code in every form: run in the portable one")] fn ecdh_sec1() { use toyos_crypto::ecc::{p256, p384}; for (file, width) in [("ecdh_secp256r1_ecpoint_test.json", 32), ("ecdh_secp384r1_ecpoint_test.json", 48)] { diff --git a/lib/ssh/src/kex.rs b/lib/ssh/src/kex.rs index e108c2afa1a..fde38f6d5db 100644 --- a/lib/ssh/src/kex.rs +++ b/lib/ssh/src/kex.rs @@ -133,6 +133,7 @@ pub(crate) struct Exchange<'a> { impl Exchange<'_> { /// `H`. pub(crate) fn hash(&self) -> [u8; 32] { + let _protection = toyos_crypto::timing::protect(); let mut data = Vec::new(); for field in [ self.client_version, @@ -154,6 +155,7 @@ impl Exchange<'_> { /// session_id)`, extended by `HASH(K || H || K1)`. `secret` is `K` encoded as /// in the exchange hash. pub(crate) fn derive(secret: &[u8], hash: &[u8; 32], letter: u8, session_id: &[u8; 32]) -> [u8; 64] { + let _protection = toyos_crypto::timing::protect(); let first = sha256(&[secret, hash, &[letter], session_id]); let second = sha256(&[secret, hash, &first]); let mut key = [0; 64]; diff --git a/lib/ssh/src/wire.rs b/lib/ssh/src/wire.rs index 4cd4f14d332..a9d8dc6137b 100644 --- a/lib/ssh/src/wire.rs +++ b/lib/ssh/src/wire.rs @@ -203,15 +203,20 @@ pub(crate) fn put_string(out: &mut Vec, bytes: &[u8]) { /// An `mpint` holding the unsigned big-endian integer `magnitude`: no leading /// zero bytes, and one zero byte where the top bit would read as a sign -/// (RFC 4251 §5). +/// (RFC 4251 §5). `magnitude` may be secret, as K is: the scan for its first +/// nonzero byte branches on no byte and reads every one. What the encoding's +/// length says of it, the format publishes. pub(crate) fn put_mpint(out: &mut Vec, magnitude: &[u8]) { - let first = magnitude.iter().position(|&b| b != 0).unwrap_or(magnitude.len()); - let digits = &magnitude[first..]; - let sign = digits.first().is_some_and(|&b| b & 0x80 != 0); - put_u32(out, u32::try_from(digits.len() + usize::from(sign)).expect("an mpint the server writes fits a u32")); - if sign { - out.push(0); + let (mut zeros, mut sign, mut seen) = (0usize, 0usize, 0usize); + for &byte in magnitude { + let nonzero = (usize::from(byte) | usize::from(byte).wrapping_neg()) >> (usize::BITS - 1); + sign |= nonzero & !seen & usize::from(byte >> 7); + seen |= nonzero; + zeros += 1 - seen; } + let digits = &magnitude[zeros..]; + put_u32(out, u32::try_from(digits.len() + sign).expect("an mpint the server writes fits a u32")); + out.resize(out.len() + sign, 0); out.extend_from_slice(digits); } @@ -231,6 +236,11 @@ mod tests { assert_eq!(mpint(&[0x09, 0xa3, 0x78, 0xf9, 0xb2, 0xe3, 0x32, 0xa7]), [0, 0, 0, 8, 0x09, 0xa3, 0x78, 0xf9, 0xb2, 0xe3, 0x32, 0xa7]); assert_eq!(mpint(&[0x80]), [0, 0, 0, 2, 0, 0x80]); assert_eq!(mpint(&[0, 0, 0x7f]), [0, 0, 0, 1, 0x7f]); + // A zero after the first nonzero byte is a digit, and only the first + // nonzero byte's top bit asks for the sign byte. + assert_eq!(mpint(&[0, 0x01, 0x00, 0x80]), [0, 0, 0, 3, 0x01, 0x00, 0x80]); + assert_eq!(mpint(&[0, 0, 0xff, 0x00]), [0, 0, 0, 3, 0, 0xff, 0x00]); + assert_eq!(mpint(&[]), [0, 0, 0, 0]); } #[test] diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 390a8bca26b..f5e609d1298 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -373,6 +373,9 @@ pub const SYS_CURRENT_CPU: u64 = 163; /// Map a firmware window a claim was granted at its mint. See /// [`device_firmware_window`]. pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; +/// Whether the calling thread can set its own data-independent timing. See +/// [`timing_control`]. +pub const SYS_TIMING_CONTROL: u64 = 165; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -380,13 +383,13 @@ pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 166; +pub const SYSCALL_PROFILE_BINS: usize = 167; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_DEVICE_FIRMWARE_WINDOW < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_TIMING_CONTROL < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -2676,6 +2679,12 @@ pub fn cpu_count() -> u32 { syscall(SYS_CPU_COUNT, 0, 0, 0, 0) as u32 } +/// Whether every CPU has FEAT_DIT, so a thread may set its own +/// `PSTATE.DIT`; false on x86-64. `toyos::timing` is the interface. +pub fn timing_control() -> bool { + syscall(SYS_TIMING_CONTROL, 0, 0, 0, 0) != 0 +} + /// `len` bytes of private memory, rounded up to whole [`PAGE_SIZE`]s, placed /// as `at` says. Any `prot` but `None` commits the pages: they read as zeros /// and are populated as they are touched. `None` reserves them, and diff --git a/sdk/toyos/src/arch/aarch64.rs b/sdk/toyos/src/arch/aarch64.rs index b7dc3a064df..c8682c2bf89 100644 --- a/sdk/toyos/src/arch/aarch64.rs +++ b/sdk/toyos/src/arch/aarch64.rs @@ -75,3 +75,59 @@ pub(crate) unsafe fn zero(dst: *mut u8, len: usize) { ); } } + +/// `crate::timing`'s mode: `PSTATE.DIT`, through the `DIT` register's view +/// (`S3_3_C4_C2_5`, bit 24), which assembles without FEAT_DIT's name. +pub(crate) mod timing { + use core::sync::atomic::{AtomicU8, Ordering::Relaxed}; + + /// Whether the machine has FEAT_DIT, once a thread has asked the kernel: + /// `DIT` is an undefined register without it. A machine fact, so every + /// copy of this crate in a process may keep its own. + static SUPPORT: AtomicU8 = AtomicU8::new(UNKNOWN); + const UNKNOWN: u8 = 0; + const ABSENT: u8 = 1; + const PRESENT: u8 = 2; + + const DIT: u64 = 1 << 24; + + /// That every CPU has FEAT_DIT, so `DIT` may be read and written. + #[derive(Clone, Copy)] + pub(crate) struct Mode(()); + + /// The mode, `None` without FEAT_DIT. + #[inline] + pub(crate) fn mode() -> Option { + let present = match SUPPORT.load(Relaxed) { + PRESENT => true, + ABSENT => false, + _ => { + let present = toyos_abi::syscall::timing_control(); + SUPPORT.store(if present { PRESENT } else { ABSENT }, Relaxed); + present + } + }; + present.then_some(Mode(())) + } + + impl Mode { + /// The thread's `PSTATE.DIT`. + #[inline] + pub(crate) fn is_on(self) -> bool { + let value: u64; + // SAFETY: `DIT` is readable at EL0 on a CPU with FEAT_DIT, which + // a `Mode` says every CPU has. + unsafe { core::arch::asm!("mrs {}, s3_3_c4_c2_5", out(reg) value, options(nomem, nostack, preserves_flags)) }; + value & DIT != 0 + } + + /// Set the thread's `PSTATE.DIT`. Not `nomem`, so no memory access of + /// the bracketed work moves across it; work held in registers alone + /// still may. + #[inline] + pub(crate) fn set(self, on: bool) { + // SAFETY: as `is_on`'s; the write changes `PSTATE.DIT` alone. + unsafe { core::arch::asm!("msr s3_3_c4_c2_5, {}", in(reg) if on { DIT } else { 0 }, options(nostack, preserves_flags)) }; + } + } +} diff --git a/sdk/toyos/src/arch/mod.rs b/sdk/toyos/src/arch/mod.rs index 17828c67eca..63ccb62aab5 100644 --- a/sdk/toyos/src/arch/mod.rs +++ b/sdk/toyos/src/arch/mod.rs @@ -3,9 +3,9 @@ #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "aarch64")] -pub(crate) use aarch64::{copy, zero}; +pub(crate) use aarch64::{copy, timing, zero}; #[cfg(target_arch = "x86_64")] mod x86_64; #[cfg(target_arch = "x86_64")] -pub(crate) use x86_64::{copy, zero}; +pub(crate) use x86_64::{copy, timing, zero}; diff --git a/sdk/toyos/src/arch/x86_64.rs b/sdk/toyos/src/arch/x86_64.rs index ea5159a6916..185da368621 100644 --- a/sdk/toyos/src/arch/x86_64.rs +++ b/sdk/toyos/src/arch/x86_64.rs @@ -35,3 +35,25 @@ pub(crate) unsafe fn zero(dst: *mut u8, len: usize) { ); } } + +/// `crate::timing`'s mode: none a thread sets on x86-64, so no [`Mode`] +/// exists. +pub(crate) mod timing { + #[derive(Clone, Copy)] + pub(crate) enum Mode {} + + #[inline(always)] + pub(crate) fn mode() -> Option { + None + } + + impl Mode { + pub(crate) fn is_on(self) -> bool { + match self {} + } + + pub(crate) fn set(self, _on: bool) { + match self {} + } + } +} diff --git a/sdk/toyos/src/lib.rs b/sdk/toyos/src/lib.rs index ac796157cc1..c0a88a33a39 100644 --- a/sdk/toyos/src/lib.rs +++ b/sdk/toyos/src/lib.rs @@ -43,6 +43,7 @@ pub mod spawn; pub mod syscap; pub mod sync; pub mod system; +pub mod timing; pub mod transport; pub mod volatile; diff --git a/sdk/toyos/src/timing.rs b/sdk/toyos/src/timing.rs new file mode 100644 index 00000000000..dbed7fa4039 --- /dev/null +++ b/sdk/toyos/src/timing.rs @@ -0,0 +1,60 @@ +//! Timing protection for the calling thread, scoped. [`protect`] turns on the +//! CPU's data-independent timing mode where the thread can set one, and +//! answers a [`Protection`] that puts back the state it found when it drops: +//! protections nest, and an early return or an unwinding panic restores the +//! state. Any program may use it; it asks for no right. +//! +//! The mode makes the instructions its vendor lists take time independent of +//! their data operands, and nothing more: code must already be constant-time +//! in its branches, addresses and choice of instructions. On AArch64 it is +//! `PSTATE.DIT` (FEAT_DIT), which the thread writes itself, so a protection +//! costs a few instructions. x86-64 has none a thread sets +//! (`toyos_abi::syscall::timing_control`), and there +//! [`Protection::is_active`] is false: nothing here reports a machine without +//! the mode as protected. +//! +//! Protections drop in the reverse of the order they were made, as scopes +//! end: dropping an outer one first ends the protection while an inner one is +//! alive. A protection is its thread's, so it is not `Send`. + +use core::marker::PhantomData; + +use crate::arch::timing; + +/// The calling thread's timing protection until this drops. See the module. +#[must_use = "the protection ends when this drops"] +pub struct Protection { + active: bool, + /// The mode to turn off again, where it was off before this. + restore: Option, + _thread: PhantomData<*const ()>, +} + +impl Protection { + /// Whether the mode is on for this protection's life: false where the + /// machine has none. + pub fn is_active(&self) -> bool { + self.active + } +} + +/// Turn the calling thread's timing protection on where the machine has it, +/// until the answer drops. +#[inline] +pub fn protect() -> Protection { + let mode = timing::mode(); + let restore = mode.filter(|mode| !mode.is_on()); + if let Some(mode) = restore { + mode.set(true); + } + Protection { active: mode.is_some(), restore, _thread: PhantomData } +} + +impl Drop for Protection { + #[inline] + fn drop(&mut self) { + if let Some(mode) = self.restore { + mode.set(false); + } + } +} diff --git a/tests/guest/rust-tests/Cargo.lock b/tests/guest/rust-tests/Cargo.lock index d7a211cf46b..55d77d60806 100644 --- a/tests/guest/rust-tests/Cargo.lock +++ b/tests/guest/rust-tests/Cargo.lock @@ -948,6 +948,9 @@ version = "0.16.0" [[package]] name = "toyos-crypto" version = "0.1.0" +dependencies = [ + "toyos", +] [[package]] name = "toyos-device-memory" diff --git a/tests/guest/rust-tests/src/arch/dit.rs b/tests/guest/rust-tests/src/arch/dit.rs new file mode 100644 index 00000000000..79336e5a866 --- /dev/null +++ b/tests/guest/rust-tests/src/arch/dit.rs @@ -0,0 +1,20 @@ +//! The calling thread's `PSTATE.DIT`, read through the `DIT` register's view +//! (`S3_3_C4_C2_5`, bit 24), where the kernel answers that every CPU has +//! FEAT_DIT: `None` otherwise, and on x86-64, which has no such state. + +#[cfg(target_arch = "aarch64")] +pub fn dit() -> Option { + if !toyos_abi::syscall::timing_control() { + return None; + } + let value: u64; + // SAFETY: `DIT` is readable at EL0 on a CPU with FEAT_DIT, which the + // kernel answered every CPU has. + unsafe { core::arch::asm!("mrs {}, s3_3_c4_c2_5", out(reg) value, options(nomem, nostack, preserves_flags)) }; + Some(value & 1 << 24 != 0) +} + +#[cfg(target_arch = "x86_64")] +pub fn dit() -> Option { + None +} diff --git a/tests/guest/rust-tests/src/bin/timing_protection.rs b/tests/guest/rust-tests/src/bin/timing_protection.rs new file mode 100644 index 00000000000..9c432086cee --- /dev/null +++ b/tests/guest/rust-tests/src/bin/timing_protection.rs @@ -0,0 +1,205 @@ +//! A thread's timing protection (`toyos::timing`): on AArch64 it is the +//! thread's own `PSTATE.DIT` on whichever CPU the thread runs, and the scoped +//! interface puts back what it found. +//! +//! **A guest, because the subject is the kernel's exception path**: EL0 sets +//! `PSTATE.DIT`, and only the kernel's save of `SPSR_EL1` on every exception +//! from EL0 and its return carry it across switches and migrations. No host +//! test reaches either. The state is read here with `MRS`, not through the +//! interface under test. +//! +//! - **Scoped**: a protection turns the state on and puts back what it found, +//! nested ones change nothing, and an early return and an unwinding panic +//! end it. A program asks for none of it by any right, and `toyos-crypto` +//! holds one while its secret work runs: a P-256 key's generation asks its +//! caller for entropy inside it. +//! - **Each thread's own**: [`THREADS_PER_CPU`] threads a CPU, every other +//! one protected, each spinning and sleeping [`ROUNDS`] times and reading +//! its state after each. Every reading is the thread's own, and the +//! scheduler's diary shows the premise: a protected thread picked on more +//! than one CPU, and a CPU picking one of these threads after one in the +//! other state. +//! +//! `virt_smp` runs it on AArch64, whose emulated CPUs have FEAT_DIT. + +#[path = "../arch/dit.rs"] +mod arch; + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use arch::dit as state; +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::syscap::{SysCap, TraceCursor}; +use toyos_abi::syscall; +use toyos_abi::trace::TraceRecord; +use toyos_trace::{Entry, Event, Thread}; + +const THREADS_PER_CPU: usize = 2; +const ROUNDS: usize = 200; +/// How long a round spins before it sleeps: long enough that a CPU with more +/// threads than it can run preempts some. +const SPIN: Duration = Duration::from_micros(300); +const SLEEP: Duration = Duration::from_micros(200); + +fn main() { + let supported = state().is_some(); + assert_eq!(state(), supported.then_some(false), "a thread that asked for nothing must start unprotected"); + scoped(supported); + crypto_holds_one(supported); + let (migrated, changes) = each_threads_own(supported); + if supported { + println!( + "timing_protection: PSTATE.DIT followed {} threads across {changes} switches between states, \ + and {migrated} protected threads ran on more than one CPU; nesting and early exits restore it", + threads(), + ); + } else { + println!("timing_protection: no timing protection on this machine, and none reported"); + } +} + +/// Nesting, an early return and a panic each leave the state they found. +fn scoped(supported: bool) { + let off = supported.then_some(false); + let on = supported.then_some(true); + { + let outer = toyos::timing::protect(); + assert_eq!(outer.is_active(), supported, "a protection's activity is not the machine's support"); + assert_eq!(state(), on); + { + let inner = toyos::timing::protect(); + assert_eq!(inner.is_active(), supported); + assert_eq!(state(), on); + } + assert_eq!(state(), on, "an inner protection's end ended the outer one"); + } + assert_eq!(state(), off, "the outer protection's end left the state on"); + + fn early(fail: bool) -> Result<(), ()> { + let _protection = toyos::timing::protect(); + if fail { + Err(())?; + } + Ok(()) + } + assert_eq!(early(true), Err(())); + assert_eq!(state(), off, "a `?` out of a protected scope left the state on"); + + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let unwound = std::panic::catch_unwind(|| { + let _protection = toyos::timing::protect(); + panic!("a panic inside a protection, caught on purpose"); + }); + std::panic::set_hook(hook); + assert!(unwound.is_err()); + assert_eq!(state(), off, "a panic out of a protected scope left the state on"); +} + +/// Entropy whose every fill reads the state it is asked in. +struct Probe(Vec>); + +impl toyos_crypto::Random for Probe { + fn fill(&mut self, out: &mut [u8]) -> Result<(), toyos_crypto::Error> { + self.0.push(state()); + out.fill(7); + Ok(()) + } +} + +/// `toyos-crypto`'s secret work runs protected, and its protection nests in +/// its caller's. +fn crypto_holds_one(supported: bool) { + let mut probe = Probe(Vec::new()); + toyos_crypto::ecc::p256::SecretKey::generate(&mut probe).expect("a scalar of sevens is below n"); + { + let _batch = toyos::timing::protect(); + toyos_crypto::ecc::p256::SecretKey::generate(&mut probe).expect("a scalar of sevens is below n"); + assert_eq!(state(), supported.then_some(true), "toyos-crypto's protection ended its caller's"); + } + assert_eq!(probe.0, [supported.then_some(true); 2], "toyos-crypto drew its secret unprotected"); + assert_eq!(state(), supported.then_some(false), "toyos-crypto left the state on"); +} + +fn threads() -> usize { + syscall::cpu_count() as usize * THREADS_PER_CPU +} + +/// Every other thread protected, each reading its own state after every +/// spin and sleep. Answers how many protected threads the diary shows on more +/// than one CPU, and how many times a CPU picked one of these threads after +/// one in the other state. +fn each_threads_own(supported: bool) -> (usize, usize) { + if !supported { + return (0, 0); + } + let mut cursor = TraceCursor::new(vec![0; cap().trace_cursor_words().expect("the estate's capability sizes a cursor")]); + read_to_now(&mut cursor); + let protected: Arc>> = Arc::default(); + let workers: Vec<_> = (0..threads()) + .map(|i| { + let protected = Arc::clone(&protected); + std::thread::spawn(move || { + let wants = i % 2 == 0; + protected.lock().unwrap().insert(toyos_abi::current_tid().raw(), wants); + let protection = wants.then(toyos::timing::protect); + for round in 0..ROUNDS { + let begun = Instant::now(); + while begun.elapsed() < SPIN { + std::hint::spin_loop(); + } + std::thread::sleep(SLEEP); + assert_eq!(state(), Some(wants), "thread {i}, which asked for {wants}, read another state in round {round}"); + } + drop(protection); + assert_eq!(state(), Some(false), "thread {i} ended its protection and read it on"); + }) + }) + .collect(); + for worker in workers { + worker.join().expect("a worker read another thread's state"); + } + + let pid = std::process::id(); + let protected = protected.lock().unwrap(); + let mut cpus: BTreeMap> = BTreeMap::new(); + let mut last: BTreeMap = BTreeMap::new(); + let mut changes = 0; + for entry in read_to_now(&mut cursor) { + let Some(Thread { pid: p, tid }) = entry.thread else { continue }; + let Some(&wants) = protected.get(&tid).filter(|_| p == pid && entry.event == Event::Pick) else { continue }; + if wants { + cpus.entry(tid).or_default().insert(entry.cpu); + } + if last.insert(entry.cpu, wants).is_some_and(|before| before != wants) { + changes += 1; + } + } + let migrated = cpus.values().filter(|on| on.len() > 1).count(); + if syscall::cpu_count() > 1 { + assert!(migrated > 0, "no protected thread was picked on a second CPU, so no migration was tested: {cpus:?}"); + } + assert!(changes > 0, "no CPU picked a thread after one in the other state, so no switch between them was tested"); + (migrated, changes) +} + +/// The estate's capability, taken once. +fn cap() -> &'static SysCap { + static CAP: std::sync::OnceLock = std::sync::OnceLock::new(); + CAP.get_or_init(|| Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a capability")) +} + +/// Every record from `cursor` on, until a read leaves its buffer short. +fn read_to_now(cursor: &mut TraceCursor>) -> Vec { + let mut out = Vec::new(); + let mut raw = vec![TraceRecord::EMPTY; 512]; + loop { + let n = cap().trace(cursor, &mut raw).expect("the estate's capability reads the diary"); + out.extend(raw[..n].iter().filter_map(|r| Entry::decode(r).ok())); + if n < raw.len() { + return out; + } + } +} diff --git a/tests/images/virtsmpcase.toml b/tests/images/virtsmpcase.toml index b71ae7ee639..db2548b8708 100644 --- a/tests/images/virtsmpcase.toml +++ b/tests/images/virtsmpcase.toml @@ -1,8 +1,10 @@ # Every CPU the machine names, and the jobs whose premise is more than one # CPU: a page unmapped beside a thread that is still reading it; a counters read -# every CPU answers for itself, the suite's `test_rs_counters_read`; and a diary -# every CPU writes, the suite's `test_rs_trace_read`. The harness puts both on -# ROOT. Then the machine powers off, every CPU but one turned off first. +# every CPU answers for itself, the suite's `test_rs_counters_read`; a diary +# every CPU writes, the suite's `test_rs_trace_read`; and each thread's own +# `PSTATE.DIT` as it moves between them, the suite's +# `test_rs_timing_protection`. The harness puts all three on ROOT. Then the +# machine powers off, every CPU but one turned off first. [boot] start = ["logkeeper", "test-runner"] @@ -14,9 +16,9 @@ syscap = ["logread"] receives = ["power"] # Four minutes from boot: every job here runs on emulated CPUs, and a job past # the bound reboots the guest with the job named. -args = ["--bound-ms=240000", "unmap_touch", "test_rs_counters_read", "test_rs_trace_read", "shutdown"] +args = ["--bound-ms=240000", "unmap_touch", "test_rs_counters_read", "test_rs_trace_read", "test_rs_timing_protection", "shutdown"] # What `test_rs_counters_read` and `test_rs_trace_read` read, and narrow away -# to prove their refusals. +# to prove their refusals, and the diary `test_rs_timing_protection` reads. syscap = ["dup", "counters", "trace"] [programs.kernelprobe] diff --git a/tests/toyos.rs b/tests/toyos.rs index 4f33f1c52b2..d14260b77bf 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -97,6 +97,10 @@ const RUST_SKIP: &[&str] = &[ // Its product is a cycle count per syscall and the clock rate beside it, // which a guest's host sets: the `syscall_cost` metal row runs it. "syscall_cost", + // Its subject is AArch64's `PSTATE.DIT`, of which x86-64 has nothing: + // `virt_smp` builds it for AArch64 and runs it on that architecture's SMP + // case. + "timing_protection", // Its product is how long a spawn of `toybox` takes to its exit, which a // guest's host sets: the `spawn_cost` metal row runs it. "spawn_cost", @@ -2738,6 +2742,9 @@ const VIRT_COUNTERS_READ: &str = "counters_read"; /// `test_rs_trace_read`. const VIRT_TRACE_READ: &str = "trace_read"; +/// The same for its job `test_rs_timing_protection`. +const VIRT_TIMING: &str = "timing_protection"; + /// `tests/guest/rust-tests`' binary `name` built for `arch`, once a run, as a /// file a case's job list names on ROOT. fn suite_bin(arch: toyos_build::arch::Arch, name: &'static str) -> (String, Vec) { @@ -3821,6 +3828,7 @@ fn boot_virt_smp(options: BootOptions) -> QemuInstance { extra_root_files: vec![ suite_bin(options.profile.arch(), VIRT_COUNTERS_READ), suite_bin(options.profile.arch(), VIRT_TRACE_READ), + suite_bin(options.profile.arch(), VIRT_TIMING), ], ..options }, @@ -3833,11 +3841,16 @@ const COUNTERS_READ_SAID: &str = "counters_read: every cpu answered for itself, /// What `trace_read` says once the diary read back as the scheduler wrote it. const TRACE_READ_SAID: &str = "trace_read: a wake precedes its pick, a cursor is its reader's own, and the diary is TRACE's"; +/// What `timing_protection` says once `PSTATE.DIT`, which every emulated CPU +/// of the case has, followed each of its threads. +const TIMING_DIT_SAID: &str = "timing_protection: PSTATE.DIT followed "; + /// Boot `tests/images/virtsmpcase.toml` on [`VIRT_CPUS`] CPUs under `profile`, whose /// firmware enters every CPU at EL`el` and whose FADT names PSCI's `conduit`: /// each CPU is started by `CPU_ON`, holds the control-register declaration as /// entered there and joins the scheduler, and the case's jobs `unmap_touch`, -/// `test_rs_counters_read` and `test_rs_trace_read` end with exit 0. Then its job `shutdown` stops the machine and powers it +/// `test_rs_counters_read`, `test_rs_trace_read` and `test_rs_timing_protection` +/// end with exit 0. Then its job `shutdown` stops the machine and powers it /// off, every other CPU turned off first ([`psci_powered_off`]). fn virt_smp(profile: qemu::Profile, conduit: &str, el: u32) -> Result<(), String> { let trace = common::lane::dir().join(format!("virt_smp-{conduit}.psci")); @@ -3881,6 +3894,7 @@ fn virt_smp(profile: qemu::Profile, conduit: &str, el: u32) -> Result<(), String eprintln!(" [virt] {VIRT_CPUS} CPUs entered at EL{el}, started through {conduit}, and scheduling"); judge_virt_job(&mut qemu, &mut serial, "test_rs_counters_read", COUNTERS_READ_SAID)?; judge_virt_job(&mut qemu, &mut serial, "test_rs_trace_read", TRACE_READ_SAID)?; + judge_virt_job(&mut qemu, &mut serial, "test_rs_timing_protection", TIMING_DIT_SAID)?; let (console, calls) = ended_through_psci(&mut qemu, &mut stop, serial, power::SHUTTING_DOWN, "guest-shutdown", &trace, |_| Vec::new())?; let record = console