diff --git a/.github/workflows/hevc-conformance.yml b/.github/workflows/hevc-conformance.yml index 1d416b7a6..38f4ea510 100644 --- a/.github/workflows/hevc-conformance.yml +++ b/.github/workflows/hevc-conformance.yml @@ -296,10 +296,10 @@ jobs: - name: Require AVX2 on the vector execution host if: runner.os == 'Linux' run: grep -qw avx2 /proc/cpuinfo - - name: Prepare bounded prediction differential targets + - name: Prepare bounded kernel differential targets run: | scripts/fuzz_run.sh setup - printf '%s\n' 'crates/coreav-kernels-hevc/fuzz horizontal' 'crates/coreav-kernels-hevc/fuzz vertical' 'crates/coreav-kernels-hevc/fuzz sao' 'crates/coreav-kernels-hevc/fuzz intra' > "$RUNNER_TEMP/interpolation-plan.txt" + printf '%s\n' 'crates/coreav-kernels-hevc/fuzz horizontal' 'crates/coreav-kernels-hevc/fuzz vertical' 'crates/coreav-kernels-hevc/fuzz sao' 'crates/coreav-kernels-hevc/fuzz intra' 'crates/coreav-kernels-hevc/fuzz deblock' > "$RUNNER_TEMP/interpolation-plan.txt" scripts/fuzz_run.sh build "$RUNNER_TEMP/interpolation-plan.txt" - id: differential name: Compare automatic and scalar kernels for ten minutes per target diff --git a/crates/coreav-kernels-hevc/fuzz/Cargo.toml b/crates/coreav-kernels-hevc/fuzz/Cargo.toml index a01d8e0c2..be462d27d 100644 --- a/crates/coreav-kernels-hevc/fuzz/Cargo.toml +++ b/crates/coreav-kernels-hevc/fuzz/Cargo.toml @@ -44,3 +44,10 @@ path = "fuzz_targets/intra.rs" test = false doc = false bench = false + +[[bin]] +name = "deblock" +path = "fuzz_targets/deblock.rs" +test = false +doc = false +bench = false diff --git a/crates/coreav-kernels-hevc/fuzz/fuzz_targets/deblock.rs b/crates/coreav-kernels-hevc/fuzz/fuzz_targets/deblock.rs new file mode 100644 index 000000000..129b3ccb2 --- /dev/null +++ b/crates/coreav-kernels-hevc/fuzz/fuzz_targets/deblock.rs @@ -0,0 +1,43 @@ +#![no_main] + +use coreav_kernels_hevc::{ + BitDepth, BoundaryStrength, deblock_chroma, deblock_chroma_scalar, deblock_luma_scalar, + deblock_luma_with_strength, +}; + +libfuzzer_sys::fuzz_target!(|data: &[u8]| { + let Some(parameters) = data.get(..6) else { + return; + }; + let flags = parameters[0]; + let depth = BitDepth::new(if flags & 1 == 0 { 8 } else { 10 }).unwrap(); + let qp = i32::from(parameters[1] as i8); + let offsets = [ + i32::from(parameters[2] as i8) % 8, + i32::from(parameters[3] as i8) % 8, + ]; + let strength = match parameters[4] % 3 { + 0 => BoundaryStrength::Zero, + 1 => BoundaryStrength::One, + _ => BoundaryStrength::Two, + }; + let protected = [parameters[5] & 1 != 0, parameters[5] & 2 != 0]; + let mut edge = [[0; 8]; 4]; + for (sample, bytes) in edge.iter_mut().flatten().zip(data[6..].chunks_exact(2)) { + let value = u16::from_le_bytes([bytes[0], bytes[1]]); + *sample = if flags & 2 == 0 { + value & depth.max_sample() + } else { + value + }; + } + assert_eq!( + deblock_luma_with_strength(depth, edge, qp, offsets, strength, protected), + deblock_luma_scalar(depth, edge, qp, offsets, strength, protected) + ); + let chroma = edge.map(|row| [row[2], row[3], row[4], row[5]]); + assert_eq!( + deblock_chroma(depth, chroma, qp, offsets[1], protected), + deblock_chroma_scalar(depth, chroma, qp, offsets[1], protected) + ); +}); diff --git a/crates/coreav-kernels-hevc/src/filter.rs b/crates/coreav-kernels-hevc/src/filter.rs index f04df1dd8..c190fa346 100644 --- a/crates/coreav-kernels-hevc/src/filter.rs +++ b/crates/coreav-kernels-hevc/src/filter.rs @@ -22,7 +22,7 @@ impl BoundaryStrength { } } -fn thresholds( +pub(super) fn thresholds( depth: BitDepth, qp: i32, offsets: [i32; 2], @@ -46,7 +46,7 @@ fn thresholds( )) } -fn tc(index: i32) -> i32 { +pub(super) fn tc(index: i32) -> i32 { match index.clamp(0, 53) { 0..=17 => 0, 18..=26 => 1, @@ -70,12 +70,12 @@ fn tc(index: i32) -> i32 { } } -fn curvature(line: [u16; 8]) -> [i32; 2] { +pub(super) fn curvature(line: [u16; 8]) -> [i32; 2] { let [_, p2, p1, p0, q0, q1, q2, _] = line.map(i32::from); [(p2 - 2 * p1 + p0).abs(), (q2 - 2 * q1 + q0).abs()] } -fn strong(line: [u16; 8], beta: i32, tc: i32) -> bool { +pub(super) fn strong(line: [u16; 8], beta: i32, tc: i32) -> bool { let [p3, _, _, p0, q0, _, _, q3] = line.map(i32::from); let [dp, dq] = curvature(line); 2 * (dp + dq) < beta >> 2 @@ -104,6 +104,26 @@ pub fn deblock_luma_with_strength( offsets: [i32; 2], strength: BoundaryStrength, protected: [bool; 2], +) -> Result<[[u16; 8]; 4], Error> { + #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] + let avx2 = (depth.bits() == 8) + .then(crate::simd::Avx2::detect) + .flatten(); + #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] + if let Some(avx2) = avx2 { + return avx2.deblock_luma(edge, qp, offsets, strength, protected); + } + deblock_luma_scalar(depth, edge, qp, offsets, strength, protected) +} + +#[doc(hidden)] +pub fn deblock_luma_scalar( + depth: BitDepth, + edge: [[u16; 8]; 4], + qp: i32, + offsets: [i32; 2], + strength: BoundaryStrength, + protected: [bool; 2], ) -> Result<[[u16; 8]; 4], Error> { if edge.iter().flatten().any(|v| *v > depth.max_sample()) { return Err(Error::Sample); @@ -184,6 +204,25 @@ pub fn deblock_chroma( qp: i32, tc_offset: i32, protected: [bool; 2], +) -> Result<[[u16; 4]; 4], Error> { + #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] + let avx2 = (depth.bits() == 8) + .then(crate::simd::Avx2::detect) + .flatten(); + #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] + if let Some(avx2) = avx2 { + return avx2.deblock_chroma(edge, qp, tc_offset, protected); + } + deblock_chroma_scalar(depth, edge, qp, tc_offset, protected) +} + +#[doc(hidden)] +pub fn deblock_chroma_scalar( + depth: BitDepth, + edge: [[u16; 4]; 4], + qp: i32, + tc_offset: i32, + protected: [bool; 2], ) -> Result<[[u16; 4]; 4], Error> { if !(-24..=63).contains(&qp) || !(-6..=6).contains(&tc_offset) { return Err(Error::Qp); @@ -387,6 +426,109 @@ impl SaoEdge { mod tests { use super::*; + #[test] + fn deblock_matches_scalar_across_parameters_and_protection() -> Result<(), Error> { + let patterns: [[u16; 8]; 6] = [ + [100, 100, 100, 100, 104, 104, 104, 104], + [100, 100, 100, 100, 140, 140, 140, 140], + [255, 255, 255, 254, 255, 255, 255, 255], + [0, 0, 0, 1, 0, 0, 0, 0], + [0, 255, 0, 255, 0, 255, 0, 255], + [12, 10, 14, 14, 21, 19, 18, 20], + ]; + for bits in [8, 10] { + let depth = BitDepth::new(bits)?; + for qp in -25i32..=64 { + for beta in -7i32..=7 { + for tc in -7i32..=7 { + let pattern = patterns[(qp + beta + tc).rem_euclid(6) as usize]; + let edge = core::array::from_fn(|row| { + core::array::from_fn(|column| { + let value = pattern[column] + .saturating_add(row as u16 * if column >= 4 { 2 } else { 1 }) + .min(255); + value << (bits - 8) + }) + }); + let chroma = edge.map(|line| [line[2], line[3], line[4], line[5]]); + for protection in 0..4 { + let protected = [protection & 1 != 0, protection & 2 != 0]; + for strength in [ + BoundaryStrength::Zero, + BoundaryStrength::One, + BoundaryStrength::Two, + ] { + assert_eq!( + deblock_luma_with_strength( + depth, + edge, + qp, + [beta, tc], + strength, + protected + ), + deblock_luma_scalar( + depth, + edge, + qp, + [beta, tc], + strength, + protected + ), + "{bits} {qp} {beta} {tc} {strength:?} {protected:?}" + ); + } + assert_eq!( + deblock_chroma(depth, chroma, qp, tc, protected), + deblock_chroma_scalar(depth, chroma, qp, tc, protected), + "{bits} {qp} {tc} {protected:?}" + ); + } + } + } + } + } + Ok(()) + } + + #[test] + fn deblock_matches_scalar_for_every_sample_word_and_error_order() -> Result<(), Error> { + for bits in [8, 10] { + let depth = BitDepth::new(bits)?; + for sample in 0..=u16::MAX { + let mut edge = [[0; 8]; 4]; + edge[usize::from(sample) % 4][usize::from(sample / 4) % 8] = sample; + let mut chroma = [[0; 4]; 4]; + chroma[usize::from(sample) % 4][usize::from(sample / 4) % 4] = sample; + for qp in [i32::MIN, 26, i32::MAX] { + assert_eq!( + deblock_luma_with_strength( + depth, + edge, + qp, + [0; 2], + BoundaryStrength::Two, + [false; 2] + ), + deblock_luma_scalar( + depth, + edge, + qp, + [0; 2], + BoundaryStrength::Two, + [false; 2] + ) + ); + assert_eq!( + deblock_chroma(depth, chroma, qp, 0, [false; 2]), + deblock_chroma_scalar(depth, chroma, qp, 0, [false; 2]) + ); + } + } + } + Ok(()) + } + #[test] fn sao_rows_cover_bands_offsets_tails_and_clipping() -> Result<(), Error> { for bits in [8, 10] { diff --git a/crates/coreav-kernels-hevc/src/lib.rs b/crates/coreav-kernels-hevc/src/lib.rs index 32bf07b6a..a74c63cf5 100644 --- a/crates/coreav-kernels-hevc/src/lib.rs +++ b/crates/coreav-kernels-hevc/src/lib.rs @@ -54,6 +54,9 @@ pub use filter::{ }; #[cfg(all(feature = "native-hevc", feature = "test-support"))] #[doc(hidden)] +pub use filter::{deblock_chroma_scalar, deblock_luma_scalar}; +#[cfg(all(feature = "native-hevc", feature = "test-support"))] +#[doc(hidden)] pub use inter::interpolate_inter_scalar; #[cfg(feature = "native-hevc")] pub use inter::{ diff --git a/crates/coreav-kernels-hevc/src/simd.rs b/crates/coreav-kernels-hevc/src/simd.rs index 9cc4664f7..0fbf35aa5 100644 --- a/crates/coreav-kernels-hevc/src/simd.rs +++ b/crates/coreav-kernels-hevc/src/simd.rs @@ -2,6 +2,7 @@ use crate::{Error, inter}; use core::arch::x86_64::*; use std::sync::OnceLock; +mod deblock; mod loop_filter; #[derive(Clone, Copy)] diff --git a/crates/coreav-kernels-hevc/src/simd/deblock.rs b/crates/coreav-kernels-hevc/src/simd/deblock.rs new file mode 100644 index 000000000..80f9d6600 --- /dev/null +++ b/crates/coreav-kernels-hevc/src/simd/deblock.rs @@ -0,0 +1,276 @@ +use super::Avx2; +use crate::{BitDepth, BoundaryStrength, Error, filter}; +use core::arch::x86_64::*; + +#[test] +fn detected_vector_deblocking_matches_scalar() -> Result<(), Error> { + let Some(avx2) = Avx2::detect() else { + assert!(!std::is_x86_feature_detected!("avx2")); + return Ok(()); + }; + let depth = BitDepth::new(8)?; + let edge = [ + [100, 100, 100, 100, 104, 104, 104, 104], + [103, 103, 103, 103, 108, 108, 108, 108], + [98, 98, 98, 98, 107, 107, 107, 107], + [105, 105, 105, 105, 113, 113, 113, 113], + ]; + assert_eq!( + avx2.deblock_luma(edge, 32, [0; 2], BoundaryStrength::Two, [false; 2]), + filter::deblock_luma_scalar(depth, edge, 32, [0; 2], BoundaryStrength::Two, [false; 2]) + ); + let chroma = edge.map(|line| [line[2], line[3], line[4], line[5]]); + assert_eq!( + avx2.deblock_chroma(chroma, 32, 0, [false; 2]), + filter::deblock_chroma_scalar(depth, chroma, 32, 0, [false; 2]) + ); + Ok(()) +} + +impl Avx2 { + pub(crate) fn deblock_luma( + self, + edge: [[u16; 8]; 4], + qp: i32, + offsets: [i32; 2], + strength: BoundaryStrength, + protected: [bool; 2], + ) -> Result<[[u16; 8]; 4], Error> { + unsafe { luma(edge, qp, offsets, strength, protected) } + } + + pub(crate) fn deblock_chroma( + self, + edge: [[u16; 4]; 4], + qp: i32, + tc_offset: i32, + protected: [bool; 2], + ) -> Result<[[u16; 4]; 4], Error> { + unsafe { chroma(edge, qp, tc_offset, protected) } + } +} + +#[target_feature(enable = "avx2")] +unsafe fn luma( + edge: [[u16; 8]; 4], + qp: i32, + offsets: [i32; 2], + strength: BoundaryStrength, + protected: [bool; 2], +) -> Result<[[u16; 8]; 4], Error> { + let rows = unsafe { + [ + _mm_loadu_si128(edge[0].as_ptr().cast()), + _mm_loadu_si128(edge[1].as_ptr().cast()), + _mm_loadu_si128(edge[2].as_ptr().cast()), + _mm_loadu_si128(edge[3].as_ptr().cast()), + ] + }; + let samples = _mm_or_si128( + _mm_or_si128(rows[0], rows[1]), + _mm_or_si128(rows[2], rows[3]), + ); + if _mm_testz_si128(samples, _mm_set1_epi16(-256)) == 0 { + return Err(Error::Sample); + } + let (beta, tc) = filter::thresholds(BitDepth::new(8)?, qp, offsets, strength)?; + if strength == BoundaryStrength::Zero || tc == 0 || protected == [true; 2] { + return Ok(edge); + } + let [dp0, dq0] = filter::curvature(edge[0]); + let [dp3, dq3] = filter::curvature(edge[3]); + let (dp, dq) = (dp0 + dp3, dq0 + dq3); + if dp + dq >= beta { + return Ok(edge); + } + let use_strong = filter::strong(edge[0], beta, tc) && filter::strong(edge[3], beta, tc); + let a = _mm_unpacklo_epi16(rows[0], rows[1]); + let b = _mm_unpacklo_epi16(rows[2], rows[3]); + let c = _mm_unpackhi_epi16(rows[0], rows[1]); + let d = _mm_unpackhi_epi16(rows[2], rows[3]); + let p32 = _mm_unpacklo_epi32(a, b); + let p10 = _mm_unpackhi_epi32(a, b); + let q01 = _mm_unpacklo_epi32(c, d); + let q23 = _mm_unpackhi_epi32(c, d); + let far = _mm_unpacklo_epi64(p32, _mm_srli_si128::<8>(q23)); + let outer = _mm_unpacklo_epi64(_mm_srli_si128::<8>(p32), q23); + let inner = _mm_unpacklo_epi64(p10, _mm_srli_si128::<8>(q01)); + let center = _mm_unpacklo_epi64(_mm_srli_si128::<8>(p10), q01); + let across = _mm_shuffle_epi32::<0x4e>(center); + let across_inner = _mm_shuffle_epi32::<0x4e>(inner); + let zero = _mm_setzero_si128(); + let (outer_value, inner_value, center_value) = if use_strong { + let radius = _mm_set1_epi16((2 * tc) as i16); + let mut sum = _mm_add_epi16( + _mm_slli_epi16::<1>(far), + _mm_add_epi16(_mm_slli_epi16::<1>(outer), outer), + ); + sum = _mm_add_epi16(sum, inner); + sum = _mm_add_epi16(sum, center); + sum = _mm_add_epi16(sum, across); + let outer_value = _mm_srli_epi16::<3>(_mm_add_epi16(sum, _mm_set1_epi16(4))); + let outer_value = _mm_min_epi16( + _mm_max_epi16(outer_value, _mm_sub_epi16(outer, radius)), + _mm_add_epi16(outer, radius), + ); + let sum = _mm_add_epi16(_mm_add_epi16(outer, inner), _mm_add_epi16(center, across)); + let inner_value = _mm_srli_epi16::<2>(_mm_add_epi16(sum, _mm_set1_epi16(2))); + let inner_value = _mm_min_epi16( + _mm_max_epi16(inner_value, _mm_sub_epi16(inner, radius)), + _mm_add_epi16(inner, radius), + ); + let mut sum = _mm_add_epi16(outer, _mm_slli_epi16::<1>(inner)); + sum = _mm_add_epi16(sum, _mm_slli_epi16::<1>(center)); + sum = _mm_add_epi16(sum, _mm_slli_epi16::<1>(across)); + sum = _mm_add_epi16(sum, across_inner); + let center_value = _mm_srli_epi16::<3>(_mm_add_epi16(sum, _mm_set1_epi16(4))); + let center_value = _mm_min_epi16( + _mm_max_epi16(center_value, _mm_sub_epi16(center, radius)), + _mm_add_epi16(center, radius), + ); + (outer_value, inner_value, center_value) + } else { + let primary = _mm_sub_epi16(across, center); + let neighbour = _mm_sub_epi16(across_inner, inner); + let numerator = _mm_sub_epi16( + _mm_add_epi16(_mm_slli_epi16::<3>(primary), primary), + _mm_add_epi16(_mm_slli_epi16::<1>(neighbour), neighbour), + ); + let delta = _mm_srai_epi16::<4>(_mm_add_epi16(numerator, _mm_set1_epi16(8))); + let eligible = _mm_cmpgt_epi16(_mm_set1_epi16((10 * tc) as i16), _mm_abs_epi16(delta)); + let eligible = _mm_unpacklo_epi64(eligible, eligible); + let clipped = _mm_min_epi16( + _mm_max_epi16(delta, _mm_set1_epi16(-tc as i16)), + _mm_set1_epi16(tc as i16), + ); + let delta = _mm_unpacklo_epi64(clipped, _mm_sub_epi16(zero, clipped)); + let delta = _mm_and_si128(delta, eligible); + let secondary = (beta + (beta >> 1)) >> 3; + let sides = _mm_set_epi64x( + if dq < secondary { -1 } else { 0 }, + if dp < secondary { -1 } else { 0 }, + ); + let average = _mm_srli_epi16::<1>(_mm_add_epi16( + _mm_add_epi16(outer, center), + _mm_set1_epi16(1), + )); + let correction = _mm_srai_epi16::<1>(_mm_add_epi16(_mm_sub_epi16(average, inner), delta)); + let half = (tc >> 1) as i16; + let correction = _mm_min_epi16( + _mm_max_epi16(correction, _mm_set1_epi16(-half)), + _mm_set1_epi16(half), + ); + let correction = _mm_and_si128(correction, _mm_and_si128(sides, eligible)); + ( + outer, + _mm_add_epi16(inner, correction), + _mm_add_epi16(center, delta), + ) + }; + let protect = _mm_set_epi64x( + if protected[1] { -1 } else { 0 }, + if protected[0] { -1 } else { 0 }, + ); + let maximum = _mm_set1_epi16(255); + let outer_value = _mm_min_epi16(_mm_max_epi16(outer_value, zero), maximum); + let inner_value = _mm_min_epi16(_mm_max_epi16(inner_value, zero), maximum); + let center_value = _mm_min_epi16(_mm_max_epi16(center_value, zero), maximum); + let values = [ + _mm_blendv_epi8(outer_value, outer, protect), + _mm_blendv_epi8(inner_value, inner, protect), + _mm_blendv_epi8(center_value, center, protect), + ]; + let p32 = _mm_unpacklo_epi64(far, values[0]); + let p10 = _mm_unpacklo_epi64(values[1], values[2]); + let q01 = _mm_unpackhi_epi64(values[2], values[1]); + let q23 = _mm_unpackhi_epi64(values[0], far); + let a = _mm_unpacklo_epi16(p32, p10); + let b = _mm_unpackhi_epi16(p32, p10); + let c = _mm_unpacklo_epi16(q01, q23); + let d = _mm_unpackhi_epi16(q01, q23); + let p01 = _mm_unpacklo_epi16(a, b); + let p23 = _mm_unpackhi_epi16(a, b); + let q01 = _mm_unpacklo_epi16(c, d); + let q23 = _mm_unpackhi_epi16(c, d); + let mut output = edge; + unsafe { + _mm_storeu_si128(output[0].as_mut_ptr().cast(), _mm_unpacklo_epi64(p01, q01)); + _mm_storeu_si128(output[1].as_mut_ptr().cast(), _mm_unpackhi_epi64(p01, q01)); + _mm_storeu_si128(output[2].as_mut_ptr().cast(), _mm_unpacklo_epi64(p23, q23)); + _mm_storeu_si128(output[3].as_mut_ptr().cast(), _mm_unpackhi_epi64(p23, q23)); + } + Ok(output) +} + +#[target_feature(enable = "avx2")] +unsafe fn chroma( + edge: [[u16; 4]; 4], + qp: i32, + tc_offset: i32, + protected: [bool; 2], +) -> Result<[[u16; 4]; 4], Error> { + if !(-24..=63).contains(&qp) || !(-6..=6).contains(&tc_offset) { + return Err(Error::Qp); + } + let rows = unsafe { + [ + _mm_loadl_epi64(edge[0].as_ptr().cast()), + _mm_loadl_epi64(edge[1].as_ptr().cast()), + _mm_loadl_epi64(edge[2].as_ptr().cast()), + _mm_loadl_epi64(edge[3].as_ptr().cast()), + ] + }; + let samples = _mm_or_si128( + _mm_or_si128(rows[0], rows[1]), + _mm_or_si128(rows[2], rows[3]), + ); + if _mm_testz_si128(samples, _mm_set1_epi16(-256)) == 0 { + return Err(Error::Sample); + } + let qp = crate::chroma_qp_index(qp)?; + let tc = filter::tc(qp + 2 + 2 * tc_offset) as i16; + if tc == 0 || protected == [true; 2] { + return Ok(edge); + } + let a = _mm_unpacklo_epi16(rows[0], rows[1]); + let b = _mm_unpacklo_epi16(rows[2], rows[3]); + let p10 = _mm_unpacklo_epi32(a, b); + let q01 = _mm_unpackhi_epi32(a, b); + let far = _mm_unpacklo_epi64(p10, _mm_srli_si128::<8>(q01)); + let center = _mm_unpacklo_epi64(_mm_srli_si128::<8>(p10), q01); + let across = _mm_shuffle_epi32::<0x4e>(center); + let across_far = _mm_shuffle_epi32::<0x4e>(far); + let numerator = _mm_add_epi16( + _mm_slli_epi16::<2>(_mm_sub_epi16(across, center)), + _mm_sub_epi16(far, across_far), + ); + let delta = _mm_srai_epi16::<3>(_mm_add_epi16(numerator, _mm_set1_epi16(4))); + let delta = _mm_min_epi16( + _mm_max_epi16(delta, _mm_set1_epi16(-tc)), + _mm_set1_epi16(tc), + ); + let delta = _mm_unpacklo_epi64(delta, _mm_sub_epi16(_mm_setzero_si128(), delta)); + let protect = _mm_set_epi64x( + if protected[1] { -1 } else { 0 }, + if protected[0] { -1 } else { 0 }, + ); + let value = _mm_add_epi16(center, _mm_andnot_si128(protect, delta)); + let value = _mm_min_epi16( + _mm_max_epi16(value, _mm_setzero_si128()), + _mm_set1_epi16(255), + ); + let p10 = _mm_unpacklo_epi64(far, value); + let q01 = _mm_unpackhi_epi64(value, far); + let a = _mm_unpacklo_epi16(p10, q01); + let b = _mm_unpackhi_epi16(p10, q01); + let first = _mm_unpacklo_epi16(a, b); + let last = _mm_unpackhi_epi16(a, b); + let mut output = edge; + unsafe { + _mm_storel_epi64(output[0].as_mut_ptr().cast(), first); + _mm_storel_epi64(output[1].as_mut_ptr().cast(), _mm_srli_si128::<8>(first)); + _mm_storel_epi64(output[2].as_mut_ptr().cast(), last); + _mm_storel_epi64(output[3].as_mut_ptr().cast(), _mm_srli_si128::<8>(last)); + } + Ok(output) +} diff --git a/docs/design/hevc-deblocking.md b/docs/design/hevc-deblocking.md new file mode 100644 index 000000000..cb1731ea1 --- /dev/null +++ b/docs/design/hevc-deblocking.md @@ -0,0 +1,25 @@ +# Eight-bit HEVC deblocking + +The checked luma and chroma interfaces retain their fixed four-line arrays, error order and return values. The driver still owns boundary strength, QP derivation, protection and vertical-before-horizontal traversal. Main10 remains scalar. The implementation follows ITU-T H.265 V11 clause 8.7.2 and the existing specification-written scalar equations, not another decoder. + +## Dispatch and ownership + +Use the existing cached `Avx2` capability to dispatch only eight-bit calls into the private `simd/deblock.rs` family. Its safe methods accept the same fixed-size arrays, not pointers. Luma validates samples before QP parameters; chroma retains its QP-before-sample error order. Shared safe threshold and four-line decision functions remain in `filter.rs`. Unsupported CPUs and the `scalar` feature keep scalar arithmetic; the latter compiles no SIMD module. + +The alternative was dispatch after scalar sample validation. That leaves repeated sample checks and array conversion on the hot path. Dispatching the whole checked kernel lets the bounded vector loads also validate samples, without changing the public API or moving driver control into unsafe code. Scalar references remain callable through the existing test-support feature for differential fuzzing. + +After input and parameter validation, zero clipping thresholds or two protected sides can return their unchanged edge before arithmetic. Luma's curvature, strong-filter selection and secondary-side decisions remain common scalar calculations across the four lines. They must not become independent per-lane decisions. + +## Lanes and numerical bounds + +Transpose the four rows into eight signed 16-bit lanes: four P-side samples followed by the corresponding four Q-side samples. Strong luma arithmetic computes both sides with the same weighted equations. Normal luma and chroma compute the rounded P-side delta once, then negate that rounded result for Q. Rounding an independently negated numerator would differ at ties and is not equivalent. + +All validated samples are in 0..=255. The largest strong weighted sum is 2,044, the normal luma delta numerator has magnitude at most 3,068, and the chroma numerator has magnitude at most 1,279. The clipping threshold is at most 24, so doubled thresholds, signed differences, absolute values and secondary corrections fit signed 16-bit lanes. Final sample clipping and protected-side selection preserve the scalar results. + +Luma reads and writes exactly four 16-byte rows; chroma exactly four 8-byte rows. Fixed-array types prove those extents before a pointer exists. Unaligned intrinsics impose no alignment requirement, no padding is read, no allocation is introduced and no pointer escapes. Each target-feature entry and its load/store operations gets an unsafe-inventory row under record 0045. + +## Verification and retention + +Compare automatic and scalar results across QPs, threshold offsets, boundary strengths, protection combinations, strong and normal decisions, clipping extremes and invalid samples, including error precedence. A differential fuzz target calls the checked interfaces on both valid and malformed fixed arrays. Native x86 Actions must prove runtime dispatch, and forced-scalar checks must exclude the unsafe module. Complete conformance and census checks run in both modes, along with exact-head similarity and independent review. + +Keep the vector family only after an explicit-main-versus-candidate reference comparison shows useful end-to-end benefit. An incremental dispatch control disables only this family and retains existing interpolation and SAO SIMD. These comparisons use the qualified pinned binary oracle, matched picture work, alternating repeated calls and retained no-media results. Instruction removal is only an optimistic selection tool, not a speed claim. Feature defaults, decoder isolation, source-copy accounting and other kernel families remain unchanged. diff --git a/docs/unsafe-sites.md b/docs/unsafe-sites.md index 8910e59c9..02903a64c 100644 --- a/docs/unsafe-sites.md +++ b/docs/unsafe-sites.md @@ -7,6 +7,7 @@ The HEVC interpolation family is confined by [its SIMD policy](design/hevc-simd. | `crates/coreav-kernels-hevc/Cargo.toml` | Copies every workspace lint, lowering only `unsafe_code` from forbid to deny so the private SIMD module alone can allow it. | | `crates/coreav-kernels-hevc/src/lib.rs` | Allows unsafe code only on the native-enabled, non-scalar, x86-64 private `simd` module. All other production modules forbid it. Scalar builds forbid unsafe code at the crate root. | | `crates/coreav-kernels-hevc/src/simd/loop_filter.rs` | The private runtime AVX2 capability gates the band and edge target-feature calls. Band loads and stores exactly sixteen unaligned `u16` values within its mutable row. Edge validates equal source, neighbour and output lengths before loading sixteen values per input and storing sixteen results. Complete groups end within every borrowed slice; no padding or escaped pointer is used. Safe callers validate 8-bit samples and offsets in -7..=7 before writes. Signed 16-bit comparison, addition and clipping cannot overflow that domain. Equality masks count changed samples, and safe scalar code handles every tail. Differential SAO fuzzing exercises the same checked row interfaces against the scalar sample equations. | +| `crates/coreav-kernels-hevc/src/simd/deblock.rs` | The private runtime AVX2 capability gates `luma` and `chroma`. Their fixed four-row array types prove every read and write extent before pointers are formed: four unaligned 16-byte luma loads/stores, or four unaligned 8-byte chroma loads/stores. Register masks validate all samples as eight-bit before arithmetic; luma checks samples before QP, while chroma retains QP-before-sample error precedence. Shared safe helpers validate parameters and derive thresholds. Four-line decisions remain common scalar calculations. P/Q transposition, signed-word arithmetic, rounded-delta negation, clipping and protected-side selection use only initialized register lanes. Validated sample/threshold bounds keep every intermediate in signed 16-bit range. Stores target only local fixed arrays; no padding, alignment, aliasing or escaped-pointer assumption is added. The scalar feature excludes this module, Main10 remains scalar, and differential tests/fuzzing compare the checked automatic interfaces with independent scalar twins. | | `crates/coreav-kernels-hevc/src/simd.rs` | `Avx2::detect` constructs the private capability only after cached standard-library CPU and OS AVX2 detection. `Avx2::horizontal` and `Avx2::vertical` call target-feature functions only with that capability and after checking filter lengths, shifts and complete input/output extents. `horizontal_avx2` loads exactly eight unaligned `u16` samples per tap, zero-extends to `i32`, multiplies and adds, then stores exactly eight `i32` outputs after an arithmetic shift. Its loop ends at the last complete group; every tap load ends within the checked `output.len()+filter.len()-1` samples. `vertical_avx2` loads and stores exactly eight unaligned `i32` values per complete output group; every used row covers the complete output. Pointer additions stay inside their borrowed slices. Scalar tails access only the remaining active elements. Inputs were depth-validated before output writes; normative coefficients and internal predictor bounds keep signed arithmetic in `i32`. No padding bytes are read, no alignment or aliasing beyond ordinary slice borrows is assumed, and no pointer escapes. | Every `unsafe` site in production crates, and every confined test-only `unsafe` site in the workspace, is listed here: where it is, what it does and the precondition that makes it sound. The PR that adds or changes `unsafe` adds or updates its row. Spike code is not listed; their `unsafe` never reaches production code. diff --git a/scripts/tests/test_hevc_simd_workflow.py b/scripts/tests/test_hevc_simd_workflow.py index 375617212..a601db4d9 100644 --- a/scripts/tests/test_hevc_simd_workflow.py +++ b/scripts/tests/test_hevc_simd_workflow.py @@ -21,7 +21,7 @@ def test_differential_campaign_is_bounded_and_covers_both_architectures(self): self.assertIn("if: github.event_name == 'workflow_dispatch' && !inputs.scalar", job) self.assertIn('os: [ubuntu-24.04, macos-15]', job) self.assertIn('grep -qw avx2 /proc/cpuinfo', job) - for target in ('horizontal', 'vertical', 'sao'): + for target in ('horizontal', 'vertical', 'sao', 'intra', 'deblock'): self.assertIn('crates/coreav-kernels-hevc/fuzz ' + target, job) self.assertIn('600 "$RUNNER_TEMP/fuzz"', job) self.assertIn('scripts/fuzz_upload.py "$RUNNER_TEMP/fuzz/findings"', job)