Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion crates/coreav-kernels-hevc/fuzz/fuzz_targets/combine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,17 @@ libfuzzer_sys::fuzz_target!(|data: &[u8]| {
}
}
if data[8] & 0xc0 == 0xc0 && side == usize::from(data[0] & 2 != 0) {
*buffer.last_mut().unwrap() = i32::MAX;
let index = if data[14] & 1 == 0 {
buffer.len() - 1
} else {
offset + usize::from(data[15]) % height * stride + usize::from(data[16]) % width
};
buffer[index] = match data[17] & 3 {
0 => i32::MAX,
1 => i32::MIN,
2 => minimum - 1,
_ => maximum + 1,
};
}
}
let denom = data[1] % 8;
Expand Down
44 changes: 31 additions & 13 deletions crates/coreav-kernels-hevc/src/inter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -394,28 +394,46 @@ fn combine(
let upper = (88 * max_sample) >> (depth.bits() - 8);
let minimum = (88 * lower - 24 * upper) >> 6;
let maximum = (88 * upper - 24 * lower) >> 6;
for row in first.data.chunks(first.stride).take(first.height) {
if row
.get(..first.width)
.ok_or(Error::Buffer)?
.iter()
.any(|value| !(minimum..=maximum).contains(value))
{
return Err(Error::Sample);
}
}
if let Some((_, other)) = second {
for row in other.data.chunks(other.stride).take(other.height) {
let scalar_validation = || {
for row in first.data.chunks(first.stride).take(first.height) {
if row
.get(..other.width)
.get(..first.width)
.ok_or(Error::Buffer)?
.iter()
.any(|value| !(minimum..=maximum).contains(value))
{
return Err(Error::Sample);
}
}
if let Some((_, other)) = second {
for row in other.data.chunks(other.stride).take(other.height) {
if row
.get(..other.width)
.ok_or(Error::Buffer)?
.iter()
.any(|value| !(minimum..=maximum).contains(value))
{
return Err(Error::Sample);
}
}
}
Ok(())
};
#[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))]
{
let vector = if first.width >= 8 {
super::simd::Avx2::detect()
} else {
None
};
if let Some(vector) = vector {
vector.validate_combine(first, second.map(|(_, row)| row), minimum, maximum)?;
} else {
scalar_validation()?;
}
}
#[cfg(any(feature = "scalar", not(target_arch = "x86_64")))]
scalar_validation()?;
let width = output.width;
let height = output.height;
let shift = weight.denom + 14 - depth.bits();
Expand Down
1 change: 1 addition & 0 deletions crates/coreav-kernels-hevc/src/simd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::{Error, inter};
use core::arch::x86_64::*;
use std::sync::OnceLock;

mod combine;
mod deblock;
mod loop_filter;

Expand Down
95 changes: 95 additions & 0 deletions crates/coreav-kernels-hevc/src/simd/combine.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
use super::Avx2;
use crate::{Error, Rect};
use core::arch::x86_64::*;

impl Avx2 {
pub(crate) fn validate_combine(
self,
first: Rect<'_, i32>,
second: Option<Rect<'_, i32>>,
minimum: i32,
maximum: i32,
) -> Result<(), Error> {
unsafe { validate_predictors(first, second, minimum, maximum) }
}

#[cfg(test)]
pub(crate) fn combine_samples_in_range(
self,
samples: &[i32],
minimum: i32,
maximum: i32,
) -> bool {
unsafe { samples_in_range(samples, minimum, maximum) }
}
}

#[target_feature(enable = "avx2")]
unsafe fn validate_predictors(
first: Rect<'_, i32>,
second: Option<Rect<'_, i32>>,
minimum: i32,
maximum: i32,
) -> Result<(), Error> {
for input in [Some(first), second].into_iter().flatten() {
for row in input.data.chunks(input.stride).take(input.height) {
let samples = row.get(..input.width).ok_or(Error::Buffer)?;
if !unsafe { samples_in_range(samples, minimum, maximum) } {
return Err(Error::Sample);
}
}
}
Ok(())
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn samples_in_range(samples: &[i32], minimum: i32, maximum: i32) -> bool {
let minimum_vector = _mm256_set1_epi32(minimum);
let maximum_vector = _mm256_set1_epi32(maximum);
let mut invalid = _mm256_setzero_si256();
let mut groups = samples.chunks_exact(8);
for group in groups.by_ref() {
let values = unsafe { _mm256_loadu_si256(group.as_ptr().cast()) };
invalid = _mm256_or_si256(
invalid,
_mm256_or_si256(
_mm256_cmpgt_epi32(minimum_vector, values),
_mm256_cmpgt_epi32(values, maximum_vector),
),
);
}
_mm256_movemask_epi8(invalid) == 0
&& groups
.remainder()
.iter()
.all(|value| (minimum..=maximum).contains(value))
}

#[test]
fn range_validation_checks_every_lane_without_reading_padding() {
let Some(vector) = Avx2::detect() else {
assert!(!std::is_x86_feature_detected!("avx2"));
return;
};
for (minimum, maximum) in [(-16830, 33150), (-16880, 33247)] {
for width in 0..=64 {
for offset in 0..8 {
let mut storage = std::vec![i32::MAX; offset + width];
let row = &mut storage[offset..];
for (index, value) in row.iter_mut().enumerate() {
*value = if index % 2 == 0 { minimum } else { maximum };
}
assert!(vector.combine_samples_in_range(row, minimum, maximum));
for lane in 0..width {
let saved = row[lane];
for value in [minimum - 1, maximum + 1, i32::MIN, i32::MAX] {
row[lane] = value;
assert!(!vector.combine_samples_in_range(row, minimum, maximum));
}
row[lane] = saved;
}
}
}
}
}
8 changes: 7 additions & 1 deletion docs/design/hevc-inter-combine.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,13 @@ Every admitted internal predictor has magnitude at most 33,247. Effective weight

## Dispatch boundary

Both depths use the safe row implementation. This family adds no runtime capability check, target-feature function or unsafe code. The `scalar` feature continues to exclude existing SIMD in other kernel families without changing these combination equations.
Combination arithmetic for both depths uses the safe row implementation. The `scalar` feature excludes SIMD validation and the existing SIMD in other kernel families without changing these combination equations.

## Predictor-domain validation

Range validation changes only for blocks with at least eight active samples per row, in both eight-bit and Main10 decoding. A cached private AVX2 capability selects signed comparisons over complete groups of eight `i32` predictors in `simd/combine.rs`. Dispatch occurs once before the validation loops, and one target-feature call covers both predictors. The safe scalar predicate checks every tail. Loads stay within the active borrowed row, without reading padding or writing output. Both predictors still finish validation before combination starts; shape and denominator checks retain their precedence.

Combination products, shifts, clipping and stores remain in the existing safe implementation. Unsupported CPUs, narrow rows and scalar-forced builds retain the original scalar validation loops. Full-output tests and the existing combination differential target compare valid and invalid lanes with the literal reference. Reference comparisons report every cell and any regressions; instruction counts distinguish extra shared-path work from timing or layout effects before scope decisions.

## Verification

Expand Down
1 change: 1 addition & 0 deletions docs/unsafe-sites.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ The HEVC interpolation family is confined by [its SIMD policy](design/hevc-simd.
|---|---|
| `crates/coreav-kernels-hevc/Cargo.toml` | Copies every workspace lint, lowering only `unsafe_code` from forbid to deny so the private SIMD module alone can allow it. |
| `crates/coreav-kernels-hevc/src/lib.rs` | Allows unsafe code only on the native-enabled, non-scalar, x86-64 private `simd` module. All other production modules forbid it. Scalar builds forbid unsafe code at the crate root. |
| `crates/coreav-kernels-hevc/src/simd/combine.rs` | The private cached AVX2 capability gates a block-level target-feature call covering both predictors. Each row's checked active slice supplies immutable `chunks_exact(8)` groups; every unaligned load reads exactly eight `i32` values without crossing that slice or reading padding. The functions perform no stores, pointer arithmetic or numeric predictor arithmetic. Register masks accumulate values below the inclusive minimum or above the inclusive maximum; safe scalar code checks each remaining tail. Both depth-specific bounds and all active predictors are validated before output writes. Dispatch requires a row width of at least eight; unsupported CPUs, narrow rows and scalar-forced builds retain the original scalar validation loops. Lane, alignment and exact-extent tests plus the combination differential target compare complete outputs and rejection-before-write behavior with the literal reference. |
| `crates/coreav-kernels-hevc/src/simd/loop_filter.rs` | The private runtime AVX2 capability gates the band and edge target-feature calls. Band loads and stores exactly sixteen unaligned `u16` values within its mutable row. Edge validates equal source, neighbour and output lengths before loading sixteen values per input and storing sixteen results. Complete groups end within every borrowed slice; no padding or escaped pointer is used. Safe callers validate 8-bit samples and offsets in -7..=7 before writes. Signed 16-bit comparison, addition and clipping cannot overflow that domain. Equality masks count changed samples, and safe scalar code handles every tail. Differential SAO fuzzing exercises the same checked row interfaces against the scalar sample equations. |
| `crates/coreav-kernels-hevc/src/simd/deblock.rs` | The private runtime AVX2 capability gates `luma` and `chroma`. Their fixed four-row array types prove every read and write extent before pointers are formed: four unaligned 16-byte luma loads/stores, or four unaligned 8-byte chroma loads/stores. Register masks validate all samples as eight-bit before arithmetic; luma checks samples before QP, while chroma retains QP-before-sample error precedence. Shared safe helpers validate parameters and derive thresholds. Four-line decisions remain common scalar calculations. P/Q transposition, signed-word arithmetic, rounded-delta negation, clipping and protected-side selection use only initialized register lanes. Validated sample/threshold bounds keep every intermediate in signed 16-bit range. Stores target only local fixed arrays; no padding, alignment, aliasing or escaped-pointer assumption is added. The scalar feature excludes this module, Main10 remains scalar, and differential tests/fuzzing compare the checked automatic interfaces with independent scalar twins. |
| `crates/coreav-kernels-hevc/src/simd.rs` | `Avx2::detect` constructs the private capability only after cached standard-library CPU and OS AVX2 detection. `Avx2::horizontal` and `Avx2::vertical` call target-feature functions only with that capability and after checking filter lengths, shifts and complete input/output extents. `horizontal_avx2` loads exactly eight unaligned `u16` samples per tap, zero-extends to `i32`, multiplies and adds, then stores exactly eight `i32` outputs after an arithmetic shift. Its loop ends at the last complete group; every tap load ends within the checked `output.len()+filter.len()-1` samples. `vertical_avx2` loads and stores exactly eight unaligned `i32` values per complete output group; every used row covers the complete output. Pointer additions stay inside their borrowed slices. Scalar tails access only the remaining active elements. Inputs were depth-validated before output writes; normative coefficients and internal predictor bounds keep signed arithmetic in `i32`. No padding bytes are read, no alignment or aliasing beyond ordinary slice borrows is assumed, and no pointer escapes. |
Expand Down
Loading