diff --git a/crates/coreav-h264-encoder/src/codec.rs b/crates/coreav-h264-encoder/src/codec.rs index d1efef029..02737b2df 100644 --- a/crates/coreav-h264-encoder/src/codec.rs +++ b/crates/coreav-h264-encoder/src/codec.rs @@ -9,7 +9,7 @@ use crate::{ syntax, transform as t, vlc::{INTER_CBP, INTRA_CBP}, }; -use coreav_kernels_h264::{BlockMut, IntraAvail, IntraMut, Kernels, Scalar}; +use coreav_kernels_h264::{BlockMut, Intra4x4Neighbours, IntraAvail, IntraMut, Kernels, Scalar}; use std::sync::{Mutex, OnceLock, TryLockError}; #[path = "inter.rs"] @@ -61,6 +61,21 @@ impl Tile { } Ok(()) } + fn neighbours4(&mut self, x: usize, y: usize, avail: IntraAvail) -> Result { + let view = IntraMut::new(&mut self.0, Self::at(x, y), STRIDE, 4, avail) + .map_err(|_| Error::Internal)?; + Intra4x4Neighbours::of(&view).ok_or(Error::Internal) + } + fn put4(&mut self, x: usize, y: usize, block: &[u8; 16]) -> Result<()> { + for (j, row) in block.chunks_exact(4).enumerate() { + let at = Self::at(x, y + j); + self.0 + .get_mut(at..at + 4) + .ok_or(Error::Internal)? + .copy_from_slice(row); + } + Ok(()) + } fn forward(&self, source: &Self, x: usize, y: usize) -> Result { t::forward(&source.0, &self.0, Self::at(x, y), STRIDE) } @@ -538,11 +553,12 @@ fn intra4( let predicted = predicted_mode(&out.modes, top, left, p); let mut ranked = [UNRANKED; 9]; let mut tile = out.tile.clone(); + let neighbours = tile.neighbours4(x, y, a)?; for (slot, mode) in ranked.iter_mut().zip(0..9) { if !legal(mode, 4, a) { continue; } - tile.prediction(x, y, 4, a, mode)?; + tile.put4(x, y, &neighbours.predict(mode))?; let mut bits = Bits::count(); mode_bits(&mut bits, mode, predicted)?; *slot = ( @@ -557,7 +573,7 @@ fn intra4( .filter(|r| **r != UNRANKED) .ok_or(Error::Internal)?; cheap_score += score; - out.tile.prediction(x, y, 4, a, mode)?; + out.tile.put4(x, y, &neighbours.predict(mode))?; let levels = out.tile.code(source, x, y, quant)?; let count = cavlc::residual( &mut Bits::count(), @@ -576,7 +592,7 @@ fn intra4( break; } let result = (|| { - tile.prediction(x, y, 4, a, mode)?; + tile.put4(x, y, &neighbours.predict(mode))?; let levels = tile.code(source, x, y, quant)?; let mut bits = Bits::count(); mode_bits(&mut bits, mode, predicted)?; diff --git a/crates/coreav-kernels-h264/src/intra.rs b/crates/coreav-kernels-h264/src/intra.rs index 015731a5d..a3842069b 100644 --- a/crates/coreav-kernels-h264/src/intra.rs +++ b/crates/coreav-kernels-h264/src/intra.rs @@ -126,23 +126,23 @@ fn dc(nb: &Neighbours, n: usize) -> i32 { } } -/// Intra_4x4 prediction of a 4x4 view (clause 8.3.1.2). `mode` is -/// Intra4x4PredMode; above 8 it predicts DC. A view of another size is -/// left unchanged. -pub(crate) fn pred4x4(v: &mut IntraMut<'_>, mode: u8) { - if v.n() != 4 { - return; - } - let top = v.top(); - let last = top.get(3).copied().unwrap_or(0); - let top: [u16; 8] = core::array::from_fn(|p| u16::from(top.get(p).copied().unwrap_or(last))); - let left: [u16; 4] = core::array::from_fn(|p| u16::from(v.left(p))); +/// Intra_4x4 prediction (clause 8.3.1.2) from the neighbours of a 4x4 +/// block, row by row. `mode` is Intra4x4PredMode; above 8 it predicts +/// DC. The neighbours are read once ([`Intra4x4Neighbours::of`]), so each +/// mode of one block is predicted without reading its view again. +#[inline] +pub(crate) fn pred4x4(neighbours: &Intra4x4Neighbours, mode: u8) -> [u8; 16] { + let Intra4x4Neighbours { + top, + left, + top_left: x, + avail, + } = *neighbours; let [a, b, c, d, e, f, g, _] = top; let [i, j, k, l] = left; - let x = u16::from(v.top_left()); let half = |a: u16, b: u16| ((a + b + 1) >> 1) as u8; let quarter = |a: u16, b: u16, c: u16| ((a + 2 * b + c + 2) >> 2) as u8; - let block = match mode { + match mode { 0 => [a, b, c, d, a, b, c, d, a, b, c, d, a, b, c, d].map(|v| v as u8), 1 => [i, i, i, i, j, j, j, j, k, k, k, k, l, l, l, l].map(|v| v as u8), 3 => directional4::<3>(&top, &left, x), @@ -177,7 +177,7 @@ pub(crate) fn pred4x4(v: &mut IntraMut<'_>, mode: u8) { ] } _ => { - let dc = match (v.avail().top, v.avail().left) { + let dc = match (avail.top, avail.left) { (true, true) => (a + b + c + d + i + j + k + l + 4) >> 3, (true, false) => (a + b + c + d + 2) >> 2, (false, true) => (i + j + k + l + 2) >> 2, @@ -185,8 +185,52 @@ pub(crate) fn pred4x4(v: &mut IntraMut<'_>, mode: u8) { }; [dc as u8; 16] } - }; - write_block(v, &block); + } +} + +/// Intra_4x4 prediction of a 4x4 view (clause 8.3.1.2), as [`pred4x4`] +/// predicts it. A view of another size is left unchanged. +pub(crate) fn pred4x4_view(v: &mut IntraMut<'_>, mode: u8) { + if let Some(neighbours) = Intra4x4Neighbours::of(v) { + write_block(v, &pred4x4(&neighbours, mode)); + } +} + +/// The neighbouring samples that Intra_4x4 prediction reads (clause +/// 8.3.1.2), read once out of a 4x4 view so that each mode can be +/// predicted from them without reading the view again. +#[derive(Clone, Copy, Debug)] +pub struct Intra4x4Neighbours { + top: [u16; 8], + left: [u16; 4], + top_left: u16, + avail: IntraAvail, +} + +impl Intra4x4Neighbours { + /// The neighbours of `v`, or `None` for a view of another size. + /// Without top-right samples, p[3, -1] stands in for them. + #[inline] + pub fn of(v: &IntraMut<'_>) -> Option { + if v.n() != 4 { + return None; + } + let top = v.top(); + let last = top.get(3).copied().unwrap_or(0); + Some(Self { + top: core::array::from_fn(|p| u16::from(top.get(p).copied().unwrap_or(last))), + left: core::array::from_fn(|p| u16::from(v.left(p))), + top_left: u16::from(v.top_left()), + avail: v.avail(), + }) + } + + /// The prediction in Intra4x4PredMode `mode`, row by row; above 8 it + /// predicts DC. + #[inline] + pub fn predict(&self, mode: u8) -> [u8; 16] { + pred4x4(self, mode) + } } #[inline(always)] @@ -643,7 +687,7 @@ pub(crate) fn block_avail( } /// [`Kernels::intra4x4_mb`]: the trait's reference semantics, each block -/// predicted by [`pred4x4`] and its residual added by `k.idct4x4_add`. +/// predicted by [`pred4x4_view`] and its residual added by `k.idct4x4_add`. pub(crate) fn intra4x4_mb( k: &K, mut dst: IntraMut<'_>, @@ -660,7 +704,7 @@ pub(crate) fn intra4x4_mb( let Some(mut v) = dst.sub(x, y, 4, block_avail(mb, x, y, 4, blk)) else { continue; }; - pred4x4(&mut v, *mode); + pred4x4_view(&mut v, *mode); if nz & (1 << blk) != 0 { if let Some(b) = v.block_mut() { k.idct4x4_add(b, coeffs); @@ -759,11 +803,11 @@ mod tests { let mut p = plane(); // A 4x4 block at (1, 1) of the plane, so row 0 is above it and // column 0 is left of it. - pred4x4(&mut view(&mut p, 16, 17, 4, ALL), 0); + pred4x4_view(&mut view(&mut p, 16, 17, 4, ALL), 0); assert_eq!(p.get(2 * 16 + 1), Some(&11)); assert_eq!(p.get(16 * 4 + 1), Some(&11)); let mut p = plane(); - pred4x4(&mut view(&mut p, 16, 17, 4, ALL), 1); + pred4x4_view(&mut view(&mut p, 16, 17, 4, ALL), 1); assert_eq!(p.get(17), Some(&21)); assert_eq!(p.get(16 * 4 + 1), Some(&24)); } @@ -771,7 +815,7 @@ mod tests { #[test] fn dc_without_neighbours_is_the_mid_grey() { let mut p = vec![7u8; 16 * 16]; - pred4x4(&mut view(&mut p, 16, 0, 4, NONE), 2); + pred4x4_view(&mut view(&mut p, 16, 0, 4, NONE), 2); assert!(p.iter().take(4).all(|&s| s == 128)); let mut p = vec![7u8; 16 * 16]; Scalar.intra16x16(view(&mut p, 16, 0, 16, NONE), 2); @@ -788,7 +832,7 @@ mod tests { let v = view(p, 16, 16 * 16 + 16, n, ALL); let mut v = v; match (n, dc) { - (4, _) => pred4x4(&mut v, mode), + (4, _) => pred4x4_view(&mut v, mode), (8, 2) => pred8x8(&mut v, mode), (16, _) => pred16x16(&mut v, mode), _ => pred_chroma(&mut v, mode), @@ -807,7 +851,7 @@ mod tests { fn a_view_of_another_size_is_left_alone() { let mut p = vec![9u8; 32 * 32]; let before = p.clone(); - pred4x4(&mut view(&mut p, 32, 33, 8, ALL), 2); + pred4x4_view(&mut view(&mut p, 32, 33, 8, ALL), 2); pred8x8(&mut view(&mut p, 32, 33, 4, ALL), 2); Scalar.intra16x16(view(&mut p, 32, 33, 8, ALL), 2); Scalar.intra_chroma(view(&mut p, 32, 33, 16, ALL), 0); @@ -831,7 +875,7 @@ mod tests { #[test] fn every_prediction_matches_the_code_it_replaced() { let cases: [(&str, New, Old, usize, u8); 4] = [ - ("4x4", pred4x4, reference::pred4x4, 4, 9), + ("4x4", pred4x4_view, reference::pred4x4, 4, 9), ("8x8", pred8x8, reference::pred8x8, 8, 9), ("16x16", pred16x16, reference::pred16x16, 16, 4), ("chroma", pred_chroma, reference::pred_chroma, 8, 4), diff --git a/crates/coreav-kernels-h264/src/lib.rs b/crates/coreav-kernels-h264/src/lib.rs index e7f3591c5..b8d87d3ba 100644 --- a/crates/coreav-kernels-h264/src/lib.rs +++ b/crates/coreav-kernels-h264/src/lib.rs @@ -33,6 +33,7 @@ mod simd; mod strengths; mod views; +pub use intra::Intra4x4Neighbours; pub use predict::{MAX_ORIGIN, MAX_SIZE, Macroblocks, Ops, Part, PlaneOp, Planes, PlanesMut}; pub use scalar::Scalar; pub use simd::Simd; @@ -268,7 +269,7 @@ pub trait Kernels: Send + Sync + 'static { /// Availability includes the block-order restriction on top-right samples. fn intra4x4(&self, mut dst: IntraMut<'_>, mode: u8) { if dst.n() == 4 { - intra::pred4x4(&mut dst, mode); + intra::pred4x4_view(&mut dst, mode); } } diff --git a/docs/design/h264-encoder-pipeline.md b/docs/design/h264-encoder-pipeline.md index 766d5b4b7..3f197b5a6 100644 --- a/docs/design/h264-encoder-pipeline.md +++ b/docs/design/h264-encoder-pipeline.md @@ -63,6 +63,26 @@ The wavefront removes the consecutive motion-vector count from costs, not from a The [row-wavefront implementation](h264-encoder-wavefront.md) separates immutable macroblock decisions from serial entropy coding and deblocking. Its explicit worker-count constructor and automatic available-core selection produce the same stream. Padding and half-sample planes are not prerequisites: workers share the existing immutable reference picture and retain the existing per-call interpolation. +**Intra 4x4 neighbours once per block (identical output).** The Intra_4x4 ranking predicts up to nine modes of each 4x4 block from the same neighbours, and each prediction used to build a checked view, read those neighbours again and copy its rows through a call per row. `Intra4x4Neighbours` in the shared kernel crate now reads them once per block, and each mode's prediction is a 16-byte array that the encoder stores with fixed four-byte row copies. The ranking, the shortlist trials and the cheap path all predict from that one read; the neighbours lie outside the block, so the trials that overwrite the block cannot change them. The decoder's reference `pred4x4_view` predicts through the same type, and its aarch64 code makes the same four calls in one instruction fewer (551 against 552); the x86-64 decoder uses its own vector intra kernels and does not reach it. + +One GitHub Actions x86-64 job (AMD EPYC 7763, two physical cores and four logical CPUs) compares main at `68cbc80372fb15e9876e6c062f187b1677bb398a` with this change on the five existing lossy inputs at QP22 and QP28, sixty pictures, GOP32 and enabled deblocking. Callgrind counts the encode calls only; timing alternates the builds and keeps five calls per arm and cell after one warm-up. + +| Input and QP | Instructions removed | 1T time saved | 4T time saved | +|---|---:|---:|---:| +| A-22 | 3.79% | 3.43% | 2.92% | +| A-28 | 3.88% | 3.09% | 4.52% | +| B-22 | 2.63% | 3.56% | 2.01% | +| B-28 | 3.07% | 4.10% | 2.02% | +| C-22 | 0.98% | 1.61% | 1.97% | +| C-28 | 1.43% | 1.67% | 2.86% | +| D-22 | 1.85% | 2.56% | 1.92% | +| D-28 | 2.64% | 3.08% | 2.55% | +| E-22 | 1.24% | 1.07% | 1.33% | +| E-28 | 1.83% | 2.44% | 2.37% | +| Mean | 2.33% | 2.66% | 2.44% | + +Callgrind on C-22 shows predictions through a checked view (`pred4x4_view`) falling from 346,430 to 74,346 calls, now reached only outside the ranking, while 272,084 predictions from the shared read run at about 68 instructions each instead of about 211; `memcpy` calls fall by about 0.8 million. Every one of the 270 encodes in this run produces the same stream and reconstruction as main, and the pinned FFmpeg 9.0.2 decoder reproduces the reconstruction in all 30 decodes. An earlier job on an AMD EPYC 9V74 runner, with the prediction arithmetic in a different function, measured 2.37%, 2.76% and 2.22%. These are shared-runner reference figures, not a fixed-host speed verdict. + ### Compression efficiency | Rank | Item | Output | Depends on | Expected gain | Start |