diff --git a/crates/coreav-hevc/Cargo.toml b/crates/coreav-hevc/Cargo.toml index 8215040e4..a937eeda6 100644 --- a/crates/coreav-hevc/Cargo.toml +++ b/crates/coreav-hevc/Cargo.toml @@ -13,6 +13,7 @@ publish = false default = [] native-hevc = ["dep:sha2", "dep:md-5", "dep:coreav-kernels-hevc", "coreav-kernels-hevc/native-hevc"] scalar = ["coreav-kernels-hevc?/scalar"] +test-support = [] [dependencies] sha2 = { version = "0.10", optional = true } @@ -41,5 +42,15 @@ required-features = ["native-hevc"] name = "packets" required-features = ["native-hevc"] -[lints] -workspace = true +[lints.rust] +unsafe_code = "deny" +missing_docs = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +unwrap_used = "deny" +expect_used = "deny" +panic = "deny" +indexing_slicing = "deny" +todo = "deny" +unimplemented = "deny" diff --git a/crates/coreav-hevc/fuzz/Cargo.toml b/crates/coreav-hevc/fuzz/Cargo.toml index aea64f99a..0728310a5 100644 --- a/crates/coreav-hevc/fuzz/Cargo.toml +++ b/crates/coreav-hevc/fuzz/Cargo.toml @@ -12,7 +12,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -coreav-hevc = { path = "..", features = ["native-hevc"] } +coreav-hevc = { path = "..", features = ["native-hevc", "test-support"] } [[bin]] name = "intra" @@ -37,3 +37,10 @@ bench = false [lints.rust] unsafe_code = "forbid" + +[[bin]] +name = "cabac" +path = "fuzz_targets/cabac.rs" +test = false +doc = false +bench = false diff --git a/crates/coreav-hevc/fuzz/fuzz_targets/cabac.rs b/crates/coreav-hevc/fuzz/fuzz_targets/cabac.rs new file mode 100644 index 000000000..f1beada4b --- /dev/null +++ b/crates/coreav-hevc/fuzz/fuzz_targets/cabac.rs @@ -0,0 +1,7 @@ +#![no_main] +libfuzzer_sys::fuzz_target!(|data: &[u8]| { + if data.len() <= 4096 { + assert!(coreav_hevc::cabac_fast_enabled()); + assert!(coreav_hevc::cabac_engines_match_reference(data)); + } +}); diff --git a/crates/coreav-hevc/src/bits.rs b/crates/coreav-hevc/src/bits.rs index ddc0b6b9c..5a0916c27 100644 --- a/crates/coreav-hevc/src/bits.rs +++ b/crates/coreav-hevc/src/bits.rs @@ -15,6 +15,28 @@ impl<'a> Bits<'a> { self.pos } + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + pub(crate) fn cabac_window(&mut self, width: u32) -> Option { + if width == 0 { + return Some(0); + } + if width > 7 { + return None; + } + let end = self.pos.checked_add(width as usize)?; + let index = self.pos / 8; + let first = u16::from(*self.data.get(index)?); + let available = 8 - (self.pos % 8) as u32; + let value = if width <= available { + first >> (available - width) + } else { + let second = u16::from(*self.data.get(index.checked_add(1)?)?); + ((first << 8) | second) >> (8 + available - width) + }; + self.pos = end; + Some(value & ((1 << width) - 1)) + } + pub(crate) fn flag(&mut self) -> Result { let byte = self .data diff --git a/crates/coreav-hevc/src/cabac.rs b/crates/coreav-hevc/src/cabac.rs index a1a5f5c96..7bc3cf6cd 100644 --- a/crates/coreav-hevc/src/cabac.rs +++ b/crates/coreav-hevc/src/cabac.rs @@ -1,5 +1,10 @@ use crate::{Error, Result, bits::Bits}; +#[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] +#[allow(unsafe_code)] +#[path = "fast/mod.rs"] +mod fast; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Initialization { I, @@ -62,6 +67,18 @@ pub(crate) struct Cabac<'a, 'b> { } impl<'a, 'b> Cabac<'a, 'b> { + #[cfg(any(test, feature = "test-support"))] + pub(crate) fn snapshot(&self) -> (usize, u16, u16, bool, u8, u64) { + ( + self.position(), + self.range, + self.offset, + self.last_bit, + self.phase as u8, + *self.remaining_bin_budget, + ) + } + pub(crate) fn max_bins(bytes: usize) -> u64 { (bytes as u64) .saturating_mul(8) @@ -126,6 +143,16 @@ impl<'a, 'b> Cabac<'a, 'b> { } pub(crate) fn decision(&mut self, context: &mut Context) -> Result { + self.start_bin()?; + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + let result = fast::decision(self, context); + #[cfg(any(not(target_arch = "x86_64"), feature = "scalar"))] + let result = self.decode_decision(context); + self.record(result) + } + + #[cfg(any(test, feature = "test-support"))] + fn decision_reference(&mut self, context: &mut Context) -> Result { self.start_bin()?; let result = self.decode_decision(context); self.record(result) @@ -314,11 +341,166 @@ const TRANS_IDX_MPS: [u8; 64] = [ 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 62, 63, ]; +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub const fn cabac_fast_enabled() -> bool { + cfg!(all(target_arch = "x86_64", not(feature = "scalar"))) +} + +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub fn cabac_engines_match_reference(data: &[u8]) -> bool { + let budget = data + .iter() + .take(2) + .fold(0u64, |value, byte| (value << 8) | u64::from(*byte)); + let mut fast_budget = budget; + let mut safe_budget = budget; + let (mut fast, mut safe) = match ( + Cabac::new(data, &mut fast_budget), + Cabac::new(data, &mut safe_budget), + ) { + (Ok(fast), Ok(safe)) => (fast, safe), + (Err(fast), Err(safe)) => return fast == safe, + _ => return false, + }; + let mut fast_contexts: [Context; 64] = + std::array::from_fn(|index| Context::new(index as u8 * 4, 26)); + let mut safe_contexts = fast_contexts; + for &command in data.iter().take(1024) { + let (fast_result, safe_result) = match command & 7 { + 0..=3 => { + let index = usize::from(command >> 2); + let (Some(fast_context), Some(safe_context)) = + (fast_contexts.get_mut(index), safe_contexts.get_mut(index)) + else { + return false; + }; + ( + fast.decision(fast_context).map(u32::from), + safe.decision_reference(safe_context).map(u32::from), + ) + } + 4 => (fast.bypass().map(u32::from), safe.bypass().map(u32::from)), + 5 => ( + fast.terminate().map(u32::from), + safe.terminate().map(u32::from), + ), + 6 => ( + fast.raw_bits(u32::from(command >> 3)), + safe.raw_bits(u32::from(command >> 3)), + ), + _ => (fast.restart().map(|()| 0), safe.restart().map(|()| 0)), + }; + if fast_result != safe_result + || fast.snapshot() != safe.snapshot() + || fast_contexts != safe_contexts + { + return false; + } + } + true +} + #[cfg(test)] mod tests { use super::{Cabac, Context, Initialization}; use crate::Error; + #[test] + fn differential_sequences_preserve_prefixes_budgets_and_terminal_states() { + for pattern in [0, 0x42, 0x80, 0xfe, 0xff] { + let data = [pattern; 128]; + for length in 0..=data.len() { + assert!(super::cabac_engines_match_reference(&data[..length])); + } + } + assert_eq!( + super::cabac_fast_enabled(), + cfg!(all(target_arch = "x86_64", not(feature = "scalar"))) + ); + } + + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + #[test] + fn every_legal_decision_state_matches_the_safe_engine() { + for state in 0..64 { + for mps in [false, true] { + for range in 256..=510 { + for offset in 0..range { + let mut fast_budget = 1; + let mut safe_budget = 1; + let mut fast = Cabac { + input: crate::bits::Bits::new(&[0xa5, 0x5a]), + range, + offset, + last_bit: true, + phase: super::Phase::Active, + remaining_bin_budget: &mut fast_budget, + }; + let mut safe = Cabac { + input: crate::bits::Bits::new(&[0xa5, 0x5a]), + range, + offset, + last_bit: true, + phase: super::Phase::Active, + remaining_bin_budget: &mut safe_budget, + }; + let mut fast_context = Context { state, mps }; + let mut safe_context = fast_context; + assert_eq!( + fast.decision(&mut fast_context), + safe.decision_reference(&mut safe_context) + ); + assert_eq!(fast.snapshot(), safe.snapshot()); + assert_eq!(fast_context, safe_context); + } + } + } + } + } + + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + #[test] + fn fast_refill_preserves_partial_failures_and_invalid_contexts() { + for state in 0..=u8::MAX { + for position in 0..=16 { + for last_bit in [false, true] { + let mut fast_budget = 1; + let mut safe_budget = 1; + let mut fast = Cabac { + input: crate::bits::Bits::new(&[0xff, 0xa5]), + range: 510, + offset: 509, + last_bit, + phase: super::Phase::Active, + remaining_bin_budget: &mut fast_budget, + }; + let mut safe = Cabac { + input: crate::bits::Bits::new(&[0xff, 0xa5]), + range: 510, + offset: 509, + last_bit, + phase: super::Phase::Active, + remaining_bin_budget: &mut safe_budget, + }; + fast.input.skip(position).unwrap(); + safe.input.skip(position).unwrap(); + let mut fast_context = Context { state, mps: false }; + let mut safe_context = fast_context; + assert_eq!( + fast.decision(&mut fast_context), + safe.decision_reference(&mut safe_context) + ); + assert_eq!(fast.snapshot(), safe.snapshot()); + assert_eq!(fast_context, safe_context); + assert_eq!(fast.bypass(), safe.bypass()); + assert_eq!(fast.snapshot(), safe.snapshot()); + } + } + } + } + #[test] fn substream_bound_covers_the_longest_mps_runs_and_truncated_reads() { assert!( diff --git a/crates/coreav-hevc/src/fast/mod.rs b/crates/coreav-hevc/src/fast/mod.rs new file mode 100644 index 000000000..5623ade5a --- /dev/null +++ b/crates/coreav-hevc/src/fast/mod.rs @@ -0,0 +1,124 @@ +//! Decoder-private CABAC site `hevc-decision`; input access remains checked Rust. + +use core::arch::asm; + +use super::{Cabac, Context, RANGE_TAB_LPS, TRANS_IDX_LPS, TRANS_IDX_MPS}; +use crate::{Error, Result}; + +#[repr(C)] +#[derive(Clone, Copy)] +struct Entry { + lps: u16, + normalized: u16, + shift: u8, + next_lps: u8, + next_mps: u8, + reserved: u8, +} + +const _: () = assert!(core::mem::size_of::() == 8); +const _: () = assert!(core::mem::offset_of!(Entry, normalized) == 2); +const _: () = assert!(core::mem::offset_of!(Entry, shift) == 4); +const _: () = assert!(core::mem::offset_of!(Entry, next_lps) == 5); +const _: () = assert!(core::mem::offset_of!(Entry, next_mps) == 6); + +#[allow(clippy::indexing_slicing)] +const fn entries() -> [Entry; 256] { + let mut table = [Entry { + lps: 0, + normalized: 0, + shift: 0, + next_lps: 0, + next_mps: 0, + reserved: 0, + }; 256]; + let mut state = 0; + while state < 64 { + let mut column = 0; + while column < 4 { + let lps = RANGE_TAB_LPS[state][column] as u16; + let shift = lps.leading_zeros() - 7; + assert!(shift > 0); + assert!(shift <= 7); + assert!(256 + 64 * column as u16 - lps >= 128); + table[state * 4 + column] = Entry { + lps, + normalized: lps << shift, + shift: shift as u8, + next_lps: TRANS_IDX_LPS[state], + next_mps: TRANS_IDX_MPS[state], + reserved: 0, + }; + column += 1; + } + state += 1; + } + table +} + +static TABLE: [Entry; 256] = entries(); + +/// Site `hevc-decision` reads only TABLE at an index below 256. Its integer +/// instructions are baseline x86-64; it writes no memory and touches no input. +#[inline(always)] +pub(super) fn decision(decoder: &mut Cabac<'_, '_>, context: &mut Context) -> Result { + if context.state >= 64 { + return Err(Error::InvalidData("cabac_context")); + } + let index = usize::from(context.state) * 4 + usize::from((decoder.range >> 6) & 3); + let mut range = u32::from(decoder.range); + let mut offset = u32::from(decoder.offset); + let shift: u32; + let next: u32; + let lps: u8; + unsafe { + asm!( + "movzx {temporary:e}, word ptr [{table} + 8*{index}]", + "sub {range:e}, {temporary:e}", + "mov {temporary:e}, {offset:e}", + "sub {temporary:e}, {range:e}", + "setae {lps}", + "jae 2f", + "movzx {next:e}, byte ptr [{table} + 8*{index} + 6]", + "xor {shift:e}, {shift:e}", + "cmp {range:e}, 256", + "jae 3f", + "shl {range:e}, 1", + "mov {shift:e}, 1", + "jmp 3f", + "2:", + "mov {offset:e}, {temporary:e}", + "movzx {range:e}, word ptr [{table} + 8*{index} + 2]", + "movzx {shift:e}, byte ptr [{table} + 8*{index} + 4]", + "movzx {next:e}, byte ptr [{table} + 8*{index} + 5]", + "3:", + table = in(reg) TABLE.as_ptr(), + index = in(reg) index, + range = inout(reg) range, + offset = inout(reg) offset, + shift = out(reg) shift, + next = out(reg) next, + lps = out(reg_byte) lps, + temporary = out(reg) _, + options(pure, readonly, nostack), + ); + } + if shift == 0 { + decoder.range = range as u16; + context.state = next as u8; + return Ok(context.mps); + } + let Some(bits) = decoder.input.cabac_window(shift) else { + return decoder.decode_decision(context); + }; + decoder.range = range as u16; + decoder.offset = ((offset << shift) | u32::from(bits)) as u16; + decoder.last_bit = bits & 1 != 0; + if decoder.offset >= decoder.range { + return Err(Error::InvalidData("cabac_offset")); + } + let bin = context.mps ^ (lps != 0); + context.mps ^= lps != 0 && context.state == 0; + context.state = next as u8; + Ok(bin) +} diff --git a/crates/coreav-hevc/src/lib.rs b/crates/coreav-hevc/src/lib.rs index 81b3f09ad..3fd735499 100644 --- a/crates/coreav-hevc/src/lib.rs +++ b/crates/coreav-hevc/src/lib.rs @@ -7,7 +7,11 @@ //! `PictureDecoder` reconstructs complete streams with scalar inter prediction //! and presentation-order output. Production backend wiring is absent. -#![forbid(unsafe_code)] +#![deny(unsafe_code)] +#![cfg_attr( + any(feature = "scalar", not(target_arch = "x86_64")), + forbid(unsafe_code) +)] #![cfg(feature = "native-hevc")] mod bits; @@ -29,6 +33,9 @@ mod slice; mod substream; mod vui; +#[cfg(feature = "test-support")] +#[doc(hidden)] +pub use cabac::{cabac_engines_match_reference, cabac_fast_enabled}; pub use decoder::{DecodeLimits, DecodeSummary, DecodedPicture, OutputMemory, PictureDecoder}; pub use dpb::{OutputEvent, PictureOrder}; pub use intra::{IntraLimits, IntraPicture, IntraPictureDecoder, IntraPlane, IntraStats}; diff --git a/docs/design/hevc-entropy.md b/docs/design/hevc-entropy.md index 7ce7d5a67..1f9dc8ce7 100644 --- a/docs/design/hevc-entropy.md +++ b/docs/design/hevc-entropy.md @@ -21,3 +21,44 @@ CABAC arithmetic and level decoding remain unchanged in the first candidate. Consider further restructuring or assembly only after measuring this safe candidate. Any later unsafe site must retain a safe reference, document its memory preconditions and compare both implementations on arbitrary bytes. + +## Decoder-private decision engine + +Keep the current safe CABAC engine as the scalar implementation and the +differential reference. The x86-64 non-scalar decision body may use a private +`fast/` module with one named assembly site. Bin admission, caller budgets, +terminal phases, bypass, raw samples and restart stay with their existing +owners. No residual-syntax or reconstruction interface changes. + +Combine the specified LPS range, its normalized range and shift, and the two +context transitions in a compile-time table. Baseline x86-64 integer assembly +selects the decision and normalization without reading the byte source or +writing memory. Each table index is bounded before the assembly call. The +unsafe inventory states the entire table extent and every access. + +Keep the bit position in `Bits`. A checked refill reads only the actual +normalization width, at most seven bits from two bytes. Compute prospective +arithmetic state locally first. If the complete refill is unavailable, run +the safe decision body from the untouched state, preserving partial reads, +arithmetic state and the exact truncation error. After a complete refill, +commit arithmetic and input state before checking the offset, and update the +context only after successful normalization. A zero-bit refill preserves +the last consumed bit. + +Prefer this shape over a separate reservoir-backed generic engine: it keeps +one source position, avoids changing callers, and preserves the safe engine's +error-state representation. Forced scalar builds compile no assembly route. +Differential tests and fuzzing must select both bodies explicitly and compare +bins, errors, contexts, input position, range, offset, last bit, phase and +remaining budget after every operation. Check all legal arithmetic states, +short prefixes, invalid contexts, budgets, termination and restart. Complete +output checks and decoder-only one-thread comparisons remain necessary; +instruction ablations alone are not a speed result. + +Keep a separate more-probable-symbol branch instead of loading both +transitions and normalization alternatives unconditionally. When the selected +normalization width is zero, only range and context state change: return the +existing MPS without visiting the refill helper or rewriting offset and the +last consumed bit. Every LPS table entry requires at least one shift, so a +zero-width result uniquely identifies this no-refill MPS path. Refilling +decisions retain the checked-window and exact safe-fallback contract above. diff --git a/docs/unsafe-sites.md b/docs/unsafe-sites.md index 8910e59c9..b6d2da4ed 100644 --- a/docs/unsafe-sites.md +++ b/docs/unsafe-sites.md @@ -4,6 +4,9 @@ The HEVC interpolation family is confined by [its SIMD policy](design/hevc-simd. | File | Operations and safety preconditions | |---|---| +| `crates/coreav-hevc/Cargo.toml` | Copies every workspace lint, lowering only `unsafe_code` to deny for the decoder-private CABAC assembly module. | +| `crates/coreav-hevc/src/cabac.rs` | Allows unsafe code only on the private x86-64, non-scalar `fast` module. Bin admission, budget and terminal-state handling remain safe Rust. | +| `crates/coreav-hevc/src/fast/mod.rs` | Site `hevc-decision` uses baseline x86-64 integer assembly to select the arithmetic range, offset, normalization width and context transition. It reads only the immutable 256-entry TABLE. The checked context state is below 64 and the range column is masked to 0..3, so `state * 4 + column` is below 256. Each entry is exactly eight initialized bytes; size and field-offset assertions cover reads at offsets 0..1, 2..3, 4, 5 and 6. Every read ends inside the 2,048-byte static table. The block writes no memory, uses no stack and never receives or accesses the untrusted byte source. All refill accesses are checked safe Rust; incomplete refills use the unchanged safe decision body before any state mutation. Forced-scalar builds forbid unsafe code and compile no fast module. | | `crates/coreav-kernels-hevc/Cargo.toml` | Copies every workspace lint, lowering only `unsafe_code` from forbid to deny so the private SIMD module alone can allow it. | | `crates/coreav-kernels-hevc/src/lib.rs` | Allows unsafe code only on the native-enabled, non-scalar, x86-64 private `simd` module. All other production modules forbid it. Scalar builds forbid unsafe code at the crate root. | | `crates/coreav-kernels-hevc/src/simd/loop_filter.rs` | The private runtime AVX2 capability gates the band and edge target-feature calls. Band loads and stores exactly sixteen unaligned `u16` values within its mutable row. Edge validates equal source, neighbour and output lengths before loading sixteen values per input and storing sixteen results. Complete groups end within every borrowed slice; no padding or escaped pointer is used. Safe callers validate 8-bit samples and offsets in -7..=7 before writes. Signed 16-bit comparison, addition and clipping cannot overflow that domain. Equality masks count changed samples, and safe scalar code handles every tail. Differential SAO fuzzing exercises the same checked row interfaces against the scalar sample equations. |