Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions crates/coreav-h264-encoder/src/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use crate::{
syntax, transform as t,
vlc::{INTER_CBP, INTRA_CBP},
};
use coreav_kernels_h264::{BlockMut, IntraAvail, IntraMut, Kernels, Scalar};
use coreav_kernels_h264::{BlockMut, Intra4x4Neighbours, IntraAvail, IntraMut, Kernels, Scalar};
use std::sync::{Mutex, OnceLock, TryLockError};

#[path = "inter.rs"]
Expand Down Expand Up @@ -61,6 +61,21 @@ impl Tile {
}
Ok(())
}
fn neighbours4(&mut self, x: usize, y: usize, avail: IntraAvail) -> Result<Intra4x4Neighbours> {
let view = IntraMut::new(&mut self.0, Self::at(x, y), STRIDE, 4, avail)
.map_err(|_| Error::Internal)?;
Intra4x4Neighbours::of(&view).ok_or(Error::Internal)
}
fn put4(&mut self, x: usize, y: usize, block: &[u8; 16]) -> Result<()> {
for (j, row) in block.chunks_exact(4).enumerate() {
let at = Self::at(x, y + j);
self.0
.get_mut(at..at + 4)
.ok_or(Error::Internal)?
.copy_from_slice(row);
}
Ok(())
}
fn forward(&self, source: &Self, x: usize, y: usize) -> Result<t::Forward> {
t::forward(&source.0, &self.0, Self::at(x, y), STRIDE)
}
Expand Down Expand Up @@ -538,11 +553,12 @@ fn intra4(
let predicted = predicted_mode(&out.modes, top, left, p);
let mut ranked = [UNRANKED; 9];
let mut tile = out.tile.clone();
let neighbours = tile.neighbours4(x, y, a)?;
for (slot, mode) in ranked.iter_mut().zip(0..9) {
if !legal(mode, 4, a) {
continue;
}
tile.prediction(x, y, 4, a, mode)?;
tile.put4(x, y, &neighbours.predict(mode))?;
let mut bits = Bits::count();
mode_bits(&mut bits, mode, predicted)?;
*slot = (
Expand All @@ -557,7 +573,7 @@ fn intra4(
.filter(|r| **r != UNRANKED)
.ok_or(Error::Internal)?;
cheap_score += score;
out.tile.prediction(x, y, 4, a, mode)?;
out.tile.put4(x, y, &neighbours.predict(mode))?;
let levels = out.tile.code(source, x, y, quant)?;
let count = cavlc::residual(
&mut Bits::count(),
Expand All @@ -576,7 +592,7 @@ fn intra4(
break;
}
let result = (|| {
tile.prediction(x, y, 4, a, mode)?;
tile.put4(x, y, &neighbours.predict(mode))?;
let levels = tile.code(source, x, y, quant)?;
let mut bits = Bits::count();
mode_bits(&mut bits, mode, predicted)?;
Expand Down
92 changes: 68 additions & 24 deletions crates/coreav-kernels-h264/src/intra.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,23 +126,23 @@ fn dc(nb: &Neighbours, n: usize) -> i32 {
}
}

/// Intra_4x4 prediction of a 4x4 view (clause 8.3.1.2). `mode` is
/// Intra4x4PredMode; above 8 it predicts DC. A view of another size is
/// left unchanged.
pub(crate) fn pred4x4(v: &mut IntraMut<'_>, mode: u8) {
if v.n() != 4 {
return;
}
let top = v.top();
let last = top.get(3).copied().unwrap_or(0);
let top: [u16; 8] = core::array::from_fn(|p| u16::from(top.get(p).copied().unwrap_or(last)));
let left: [u16; 4] = core::array::from_fn(|p| u16::from(v.left(p)));
/// Intra_4x4 prediction (clause 8.3.1.2) from the neighbours of a 4x4
/// block, row by row. `mode` is Intra4x4PredMode; above 8 it predicts
/// DC. The neighbours are read once ([`Intra4x4Neighbours::of`]), so each
/// mode of one block is predicted without reading its view again.
#[inline]
pub(crate) fn pred4x4(neighbours: &Intra4x4Neighbours, mode: u8) -> [u8; 16] {
let Intra4x4Neighbours {
top,
left,
top_left: x,
avail,
} = *neighbours;
let [a, b, c, d, e, f, g, _] = top;
let [i, j, k, l] = left;
let x = u16::from(v.top_left());
let half = |a: u16, b: u16| ((a + b + 1) >> 1) as u8;
let quarter = |a: u16, b: u16, c: u16| ((a + 2 * b + c + 2) >> 2) as u8;
let block = match mode {
match mode {
0 => [a, b, c, d, a, b, c, d, a, b, c, d, a, b, c, d].map(|v| v as u8),
1 => [i, i, i, i, j, j, j, j, k, k, k, k, l, l, l, l].map(|v| v as u8),
3 => directional4::<3>(&top, &left, x),
Expand Down Expand Up @@ -177,16 +177,60 @@ pub(crate) fn pred4x4(v: &mut IntraMut<'_>, mode: u8) {
]
}
_ => {
let dc = match (v.avail().top, v.avail().left) {
let dc = match (avail.top, avail.left) {
(true, true) => (a + b + c + d + i + j + k + l + 4) >> 3,
(true, false) => (a + b + c + d + 2) >> 2,
(false, true) => (i + j + k + l + 2) >> 2,
(false, false) => 128,
};
[dc as u8; 16]
}
};
write_block(v, &block);
}
}

/// Intra_4x4 prediction of a 4x4 view (clause 8.3.1.2), as [`pred4x4`]
/// predicts it. A view of another size is left unchanged.
pub(crate) fn pred4x4_view(v: &mut IntraMut<'_>, mode: u8) {
if let Some(neighbours) = Intra4x4Neighbours::of(v) {
write_block(v, &pred4x4(&neighbours, mode));
}
}

/// The neighbouring samples that Intra_4x4 prediction reads (clause
/// 8.3.1.2), read once out of a 4x4 view so that each mode can be
/// predicted from them without reading the view again.
#[derive(Clone, Copy, Debug)]
pub struct Intra4x4Neighbours {
top: [u16; 8],
left: [u16; 4],
top_left: u16,
avail: IntraAvail,
}

impl Intra4x4Neighbours {
/// The neighbours of `v`, or `None` for a view of another size.
/// Without top-right samples, p[3, -1] stands in for them.
#[inline]
pub fn of(v: &IntraMut<'_>) -> Option<Self> {
if v.n() != 4 {
return None;
}
let top = v.top();
let last = top.get(3).copied().unwrap_or(0);
Some(Self {
top: core::array::from_fn(|p| u16::from(top.get(p).copied().unwrap_or(last))),
left: core::array::from_fn(|p| u16::from(v.left(p))),
top_left: u16::from(v.top_left()),
avail: v.avail(),
})
}

/// The prediction in Intra4x4PredMode `mode`, row by row; above 8 it
/// predicts DC.
#[inline]
pub fn predict(&self, mode: u8) -> [u8; 16] {
pred4x4(self, mode)
}
}

#[inline(always)]
Expand Down Expand Up @@ -643,7 +687,7 @@ pub(crate) fn block_avail(
}

/// [`Kernels::intra4x4_mb`]: the trait's reference semantics, each block
/// predicted by [`pred4x4`] and its residual added by `k.idct4x4_add`.
/// predicted by [`pred4x4_view`] and its residual added by `k.idct4x4_add`.
pub(crate) fn intra4x4_mb<K: Kernels + ?Sized>(
k: &K,
mut dst: IntraMut<'_>,
Expand All @@ -660,7 +704,7 @@ pub(crate) fn intra4x4_mb<K: Kernels + ?Sized>(
let Some(mut v) = dst.sub(x, y, 4, block_avail(mb, x, y, 4, blk)) else {
continue;
};
pred4x4(&mut v, *mode);
pred4x4_view(&mut v, *mode);
if nz & (1 << blk) != 0 {
if let Some(b) = v.block_mut() {
k.idct4x4_add(b, coeffs);
Expand Down Expand Up @@ -759,19 +803,19 @@ mod tests {
let mut p = plane();
// A 4x4 block at (1, 1) of the plane, so row 0 is above it and
// column 0 is left of it.
pred4x4(&mut view(&mut p, 16, 17, 4, ALL), 0);
pred4x4_view(&mut view(&mut p, 16, 17, 4, ALL), 0);
assert_eq!(p.get(2 * 16 + 1), Some(&11));
assert_eq!(p.get(16 * 4 + 1), Some(&11));
let mut p = plane();
pred4x4(&mut view(&mut p, 16, 17, 4, ALL), 1);
pred4x4_view(&mut view(&mut p, 16, 17, 4, ALL), 1);
assert_eq!(p.get(17), Some(&21));
assert_eq!(p.get(16 * 4 + 1), Some(&24));
}

#[test]
fn dc_without_neighbours_is_the_mid_grey() {
let mut p = vec![7u8; 16 * 16];
pred4x4(&mut view(&mut p, 16, 0, 4, NONE), 2);
pred4x4_view(&mut view(&mut p, 16, 0, 4, NONE), 2);
assert!(p.iter().take(4).all(|&s| s == 128));
let mut p = vec![7u8; 16 * 16];
Scalar.intra16x16(view(&mut p, 16, 0, 16, NONE), 2);
Expand All @@ -788,7 +832,7 @@ mod tests {
let v = view(p, 16, 16 * 16 + 16, n, ALL);
let mut v = v;
match (n, dc) {
(4, _) => pred4x4(&mut v, mode),
(4, _) => pred4x4_view(&mut v, mode),
(8, 2) => pred8x8(&mut v, mode),
(16, _) => pred16x16(&mut v, mode),
_ => pred_chroma(&mut v, mode),
Expand All @@ -807,7 +851,7 @@ mod tests {
fn a_view_of_another_size_is_left_alone() {
let mut p = vec![9u8; 32 * 32];
let before = p.clone();
pred4x4(&mut view(&mut p, 32, 33, 8, ALL), 2);
pred4x4_view(&mut view(&mut p, 32, 33, 8, ALL), 2);
pred8x8(&mut view(&mut p, 32, 33, 4, ALL), 2);
Scalar.intra16x16(view(&mut p, 32, 33, 8, ALL), 2);
Scalar.intra_chroma(view(&mut p, 32, 33, 16, ALL), 0);
Expand All @@ -831,7 +875,7 @@ mod tests {
#[test]
fn every_prediction_matches_the_code_it_replaced() {
let cases: [(&str, New, Old, usize, u8); 4] = [
("4x4", pred4x4, reference::pred4x4, 4, 9),
("4x4", pred4x4_view, reference::pred4x4, 4, 9),
("8x8", pred8x8, reference::pred8x8, 8, 9),
("16x16", pred16x16, reference::pred16x16, 16, 4),
("chroma", pred_chroma, reference::pred_chroma, 8, 4),
Expand Down
3 changes: 2 additions & 1 deletion crates/coreav-kernels-h264/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ mod simd;
mod strengths;
mod views;

pub use intra::Intra4x4Neighbours;
pub use predict::{MAX_ORIGIN, MAX_SIZE, Macroblocks, Ops, Part, PlaneOp, Planes, PlanesMut};
pub use scalar::Scalar;
pub use simd::Simd;
Expand Down Expand Up @@ -268,7 +269,7 @@ pub trait Kernels: Send + Sync + 'static {
/// Availability includes the block-order restriction on top-right samples.
fn intra4x4(&self, mut dst: IntraMut<'_>, mode: u8) {
if dst.n() == 4 {
intra::pred4x4(&mut dst, mode);
intra::pred4x4_view(&mut dst, mode);
}
}

Expand Down
20 changes: 20 additions & 0 deletions docs/design/h264-encoder-pipeline.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,26 @@ The wavefront removes the consecutive motion-vector count from costs, not from a

The [row-wavefront implementation](h264-encoder-wavefront.md) separates immutable macroblock decisions from serial entropy coding and deblocking. Its explicit worker-count constructor and automatic available-core selection produce the same stream. Padding and half-sample planes are not prerequisites: workers share the existing immutable reference picture and retain the existing per-call interpolation.

**Intra 4x4 neighbours once per block (identical output).** The Intra_4x4 ranking predicts up to nine modes of each 4x4 block from the same neighbours, and each prediction used to build a checked view, read those neighbours again and copy its rows through a call per row. `Intra4x4Neighbours` in the shared kernel crate now reads them once per block, and each mode's prediction is a 16-byte array that the encoder stores with fixed four-byte row copies. The ranking, the shortlist trials and the cheap path all predict from that one read; the neighbours lie outside the block, so the trials that overwrite the block cannot change them. The decoder's reference `pred4x4_view` predicts through the same type, and its aarch64 code makes the same four calls in one instruction fewer (551 against 552); the x86-64 decoder uses its own vector intra kernels and does not reach it.

One GitHub Actions x86-64 job (AMD EPYC 7763, two physical cores and four logical CPUs) compares main at `68cbc80372fb15e9876e6c062f187b1677bb398a` with this change on the five existing lossy inputs at QP22 and QP28, sixty pictures, GOP32 and enabled deblocking. Callgrind counts the encode calls only; timing alternates the builds and keeps five calls per arm and cell after one warm-up.

| Input and QP | Instructions removed | 1T time saved | 4T time saved |
|---|---:|---:|---:|
| A-22 | 3.79% | 3.43% | 2.92% |
| A-28 | 3.88% | 3.09% | 4.52% |
| B-22 | 2.63% | 3.56% | 2.01% |
| B-28 | 3.07% | 4.10% | 2.02% |
| C-22 | 0.98% | 1.61% | 1.97% |
| C-28 | 1.43% | 1.67% | 2.86% |
| D-22 | 1.85% | 2.56% | 1.92% |
| D-28 | 2.64% | 3.08% | 2.55% |
| E-22 | 1.24% | 1.07% | 1.33% |
| E-28 | 1.83% | 2.44% | 2.37% |
| Mean | 2.33% | 2.66% | 2.44% |

Callgrind on C-22 shows predictions through a checked view (`pred4x4_view`) falling from 346,430 to 74,346 calls, now reached only outside the ranking, while 272,084 predictions from the shared read run at about 68 instructions each instead of about 211; `memcpy` calls fall by about 0.8 million. Every one of the 270 encodes in this run produces the same stream and reconstruction as main, and the pinned FFmpeg 9.0.2 decoder reproduces the reconstruction in all 30 decodes. An earlier job on an AMD EPYC 9V74 runner, with the prediction arithmetic in a different function, measured 2.37%, 2.76% and 2.22%. These are shared-runner reference figures, not a fixed-host speed verdict.

### Compression efficiency

| Rank | Item | Output | Depends on | Expected gain | Start |
Expand Down
Loading