diff --git a/.github/workflows/h264-encoder.yml b/.github/workflows/h264-encoder.yml index 3778379fc..05900f7c8 100644 --- a/.github/workflows/h264-encoder.yml +++ b/.github/workflows/h264-encoder.yml @@ -72,7 +72,7 @@ jobs: components: clippy - name: Unit tests and opt-in driver run: | - H264_P_FIXTURE_DIR="$RUNNER_TEMP/p-forced" cargo test --locked --release -p coreav-h264-encoder --features full + H264_VUI_FIXTURE_DIR="$RUNNER_TEMP/vui" H264_P_FIXTURE_DIR="$RUNNER_TEMP/p-forced" cargo test --locked --release -p coreav-h264-encoder --features full cargo clippy --locked -p coreav-h264-encoder --features full --all-targets -- -D warnings cargo build --locked --release -p coreav-h264-encoder --features full --example g1 cargo build --locked --release -p coreav-h264-encoder --features full --example p @@ -104,6 +104,9 @@ jobs: run: | export PATH="$ORACLES:$PATH" scripts/gen-fixtures.sh "$RUNNER_TEMP/g1-existing-fixtures" + - name: Compare constant cadence and sample aspect ratio + run: | + python3 -I crates/coreav-h264-encoder/tests/vui.py --ffmpeg "$RUNNER_TEMP/g1-oracles/ffmpeg" --fixtures "$RUNNER_TEMP/vui" - name: Compare every visible plane with all three decoders run: | python3 -I scripts/h264_g1_conformance.py \ @@ -125,6 +128,7 @@ jobs: name: h264-encoder-conformance path: | ${{ runner.temp }}/g1-conformance + ${{ runner.temp }}/vui ${{ runner.temp }}/deblock-conformance ${{ runner.temp }}/g1-oracles/jm-build.txt if-no-files-found: ignore diff --git a/crates/coreav-h264-encoder/VUI.md b/crates/coreav-h264-encoder/VUI.md new file mode 100644 index 000000000..00a58ccd0 --- /dev/null +++ b/crates/coreav-h264-encoder/VUI.md @@ -0,0 +1,9 @@ +# Constant cadence and sample aspect ratio + +Implement Rec. ITU-T H.264 (08/2024), Annex E.1.1 and E.2.1, Table E-1 and Table E-6. + +Add checked `FrameRate` and `SampleAspectRatio` values to public `Settings`. Reduce positive rational inputs before checking the 32-bit timing and 16-bit Extended_SAR bounds. For progressive frames, write `num_units_in_tick = den`, `time_scale = 2 * num`, and `fixed_frame_rate_flag = 1`. When the reduced denominator is even, halve both timing fields so every representable ratio fits. Check the level 5.1 macroblocks-per-second limit against the actual rational cadence. + +Use Table E-1 for matching aspect ratios, otherwise Extended_SAR. Explicit square pixels use entry 1. The default aspect ratio preserves the existing absent signalling, which means unspecified in the specification; callers requiring explicit 1:1 use the checked constructor. The default timing remains 30/1, preserving existing encoded bytes. + +Tests independently inspect SPS fields and decode complete pictures with the native decoder, which consumes but does not retain timing or aspect metadata. A generated-fixture test compares rate and SAR with the qualified ffprobe pair on Actions. The test prepares MP4 timestamps from each emitted SPS clock, not the expected rate: the pinned raw H.264 demuxer reports a doubled rate and a 25 fps average fallback for both native and oracle-generated streams. No CLI, container cadence, variable frame rate, or decoder API changes. diff --git a/crates/coreav-h264-encoder/examples/g1.rs b/crates/coreav-h264-encoder/examples/g1.rs index 243ff468d..46adf1389 100644 --- a/crates/coreav-h264-encoder/examples/g1.rs +++ b/crates/coreav-h264-encoder/examples/g1.rs @@ -48,6 +48,8 @@ fn run() -> Result<(), Box> { let height = height.parse::()?; let frames = frames.parse::()?; let settings = Settings { + frame_rate: Default::default(), + sample_aspect_ratio: Default::default(), width, height, rate_mode: coreav_h264_encoder::RateMode::FixedQp(qp.parse()?), diff --git a/crates/coreav-h264-encoder/examples/p.rs b/crates/coreav-h264-encoder/examples/p.rs index b8d43a8b6..2e4468063 100644 --- a/crates/coreav-h264-encoder/examples/p.rs +++ b/crates/coreav-h264-encoder/examples/p.rs @@ -36,6 +36,8 @@ fn run() -> Result<(), Box> { let height = height.parse::()?; let frames = frames.parse::()?; let settings = Settings { + frame_rate: Default::default(), + sample_aspect_ratio: Default::default(), width, height, rate_mode: match qp.strip_prefix("crf:") { diff --git a/crates/coreav-h264-encoder/src/api.rs b/crates/coreav-h264-encoder/src/api.rs index 02197fdbb..19f16caee 100644 --- a/crates/coreav-h264-encoder/src/api.rs +++ b/crates/coreav-h264-encoder/src/api.rs @@ -46,9 +46,99 @@ pub struct Colour { pub chroma_location: u8, } -/// Immutable sequence settings. Timing is always nominal 30/1. +/// A positive constant frame rate, reduced to fit progressive VUI timing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameRate { + pub(crate) num: u32, + pub(crate) den: u32, +} + +impl FrameRate { + /// Reduce frames per second `num / den`; reject zero or 32-bit timing overflow. + pub fn new(num: u32, den: u32) -> Result { + let (num, den) = ratio(num, den)?; + if den % 2 != 0 { + num.checked_mul(2).ok_or(Error::Settings)?; + } + Ok(Self { num, den }) + } + + /// Reduced frames per second as (numerator, denominator). + pub fn as_ratio(self) -> (u32, u32) { + (self.num, self.den) + } + + pub(crate) fn timing(self) -> (u32, u64) { + if self.den % 2 == 0 { + (self.den / 2, u64::from(self.num)) + } else { + (self.den, 2 * u64::from(self.num)) + } + } +} + +impl Default for FrameRate { + fn default() -> Self { + Self { num: 30, den: 1 } + } +} + +/// A sample width-to-height ratio with bounded H.264 VUI signalling. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SampleAspectRatio { + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) present: bool, +} + +impl SampleAspectRatio { + /// Reduce a positive ratio; reject values exceeding 16 bits after reduction. + /// Square pixels are explicitly signalled with aspect_ratio_idc 1. + pub fn new(width: u32, height: u32) -> Result { + let (width, height) = ratio(width, height)?; + Ok(Self { + width: u16::try_from(width).map_err(|_| Error::Settings)?, + height: u16::try_from(height).map_err(|_| Error::Settings)?, + present: true, + }) + } + + /// Reduced sample width and height, including the legacy square-pixel default. + pub fn as_ratio(self) -> (u16, u16) { + (self.width, self.height) + } +} + +impl Default for SampleAspectRatio { + /// Preserve legacy bytes by omitting aspect ratio signalling (unspecified). + /// Use `new(1, 1)` when explicit square-pixel signalling is required. + fn default() -> Self { + Self { + width: 1, + height: 1, + present: false, + } + } +} + +fn ratio(num: u32, den: u32) -> Result<(u32, u32)> { + if num == 0 || den == 0 { + return Err(Error::Settings); + } + let (mut a, mut b) = (num, den); + while b != 0 { + (a, b) = (b, a % b); + } + Ok((num / a, den / a)) +} + +/// Immutable sequence settings. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Settings { + /// Constant progressive cadence; the default is 30/1 frames per second. + pub frame_rate: FrameRate, + /// Sample shape; the default preserves legacy absent signalling. + pub sample_aspect_ratio: SampleAspectRatio, /// Even visible width, 16 through 1920. pub width: usize, /// Even visible height, 16 through 1080. @@ -232,7 +322,8 @@ impl Encoder { let mbs = coded_width * coded_height / 256; if coded_width * coded_height > limits.coded_pixels || mbs > 36_864 - || mbs * 30 > 983_040 + || mbs as u64 * u64::from(settings.frame_rate.num) + > 983_040 * u64::from(settings.frame_rate.den) || mbs > 184_320 || (coded_width / 16).pow(2) > 36_864 * 8 || (coded_height / 16).pow(2) > 36_864 * 8 diff --git a/crates/coreav-h264-encoder/src/inter.rs b/crates/coreav-h264-encoder/src/inter.rs index 716be9e9d..11acd7406 100644 --- a/crates/coreav-h264-encoder/src/inter.rs +++ b/crates/coreav-h264-encoder/src/inter.rs @@ -2074,6 +2074,8 @@ mod tests { fn settings(deblocking: Deblocking) -> Settings { Settings { + frame_rate: Default::default(), + sample_aspect_ratio: Default::default(), width: 16, height: 16, rate_mode: crate::RateMode::FixedQp(28), diff --git a/crates/coreav-h264-encoder/src/rows.rs b/crates/coreav-h264-encoder/src/rows.rs index 878914a5b..8fb4e91bb 100644 --- a/crates/coreav-h264-encoder/src/rows.rs +++ b/crates/coreav-h264-encoder/src/rows.rs @@ -191,6 +191,8 @@ mod tests { #[allow(clippy::unwrap_used, clippy::indexing_slicing)] fn unavailable_wavefront_neighbours_are_refused_before_border_reads() { let settings = crate::Settings { + frame_rate: Default::default(), + sample_aspect_ratio: Default::default(), width: 64, height: 48, rate_mode: crate::RateMode::FixedQp(28), diff --git a/crates/coreav-h264-encoder/src/syntax.rs b/crates/coreav-h264-encoder/src/syntax.rs index b4e10c809..ed05b870c 100644 --- a/crates/coreav-h264-encoder/src/syntax.rs +++ b/crates/coreav-h264-encoder/src/syntax.rs @@ -24,7 +24,34 @@ pub(crate) fn sps(w: &mut Bits, e: &Encoder) -> Result<()> { w.ue(((e.coded_height - e.settings.height) / 2) as u32)?; } w.flag(true)?; - w.flag(false)?; + let sar = e.settings.sample_aspect_ratio; + w.flag(sar.present)?; + if sar.present { + let idc = match sar.as_ratio() { + (1, 1) => 1, + (12, 11) => 2, + (10, 11) => 3, + (16, 11) => 4, + (40, 33) => 5, + (24, 11) => 6, + (20, 11) => 7, + (32, 11) => 8, + (80, 33) => 9, + (18, 11) => 10, + (15, 11) => 11, + (64, 33) => 12, + (160, 99) => 13, + (4, 3) => 14, + (3, 2) => 15, + (2, 1) => 16, + _ => 255, + }; + w.u(8, idc)?; + if idc == 255 { + w.u(16, u64::from(sar.width))?; + w.u(16, u64::from(sar.height))?; + } + } w.flag(false)?; w.flag(true)?; w.u(3, 5)?; @@ -37,8 +64,9 @@ pub(crate) fn sps(w: &mut Bits, e: &Encoder) -> Result<()> { w.ue(u32::from(e.settings.colour.chroma_location))?; w.ue(u32::from(e.settings.colour.chroma_location))?; w.flag(true)?; - w.u(32, 1)?; - w.u(32, 60)?; + let (tick, scale) = e.settings.frame_rate.timing(); + w.u(32, u64::from(tick))?; + w.u(32, scale)?; w.flag(true)?; w.flag(false)?; w.flag(false)?; diff --git a/crates/coreav-h264-encoder/src/tests.rs b/crates/coreav-h264-encoder/src/tests.rs index 0766c6e49..6b86d3091 100644 --- a/crates/coreav-h264-encoder/src/tests.rs +++ b/crates/coreav-h264-encoder/src/tests.rs @@ -6,6 +6,8 @@ use bits::Bits; fn settings(width: usize, height: usize, qp: u8) -> Settings { Settings { + frame_rate: FrameRate::default(), + sample_aspect_ratio: SampleAspectRatio::default(), width, height, rate_mode: RateMode::FixedQp(qp), @@ -21,6 +23,183 @@ fn settings(width: usize, height: usize, qp: u8) -> Settings { } } +#[test] +fn rational_signalling_rejects_zero_and_unrepresentable_values() { + for (num, den) in [(0, 1), (1, 0), (0, 0), (u32::MAX, 1), (1 << 31, 1)] { + assert_eq!(FrameRate::new(num, den), Err(Error::Settings)); + } + for (num, den) in [ + (0, 1), + (1, 0), + (0, 0), + (65536, 1), + (1, 65536), + (u32::MAX, 2), + ] { + assert_eq!(SampleAspectRatio::new(num, den), Err(Error::Settings)); + } + assert_eq!( + FrameRate::new(60000, 2002).unwrap().as_ratio(), + (30000, 1001) + ); + assert_eq!( + FrameRate::new(u32::MAX, u32::MAX).unwrap().as_ratio(), + (1, 1) + ); + assert_eq!( + FrameRate::new(u32::MAX / 2, u32::MAX).unwrap().as_ratio(), + (u32::MAX / 2, u32::MAX) + ); + for (num, den, tick, scale) in [ + (u32::MAX, u32::MAX - 1, (u32::MAX - 1) / 2, u32::MAX), + (1, u32::MAX, u32::MAX, 2), + (u32::MAX / 2, u32::MAX, u32::MAX, u32::MAX - 1), + ] { + let rate = FrameRate::new(num, den).unwrap(); + assert_eq!(rate.timing(), (tick, u64::from(scale))); + let mut s = settings(16, 16, 26); + s.frame_rate = rate; + let encoder = Encoder::new(s, Limits::default()).unwrap(); + let mut bits = Bits::new(1000); + syntax::sps(&mut bits, &encoder).unwrap(); + assert!(!bits.nal(0x67).unwrap().is_empty()); + } + assert_eq!( + SampleAspectRatio::new(80000, 66000).unwrap().as_ratio(), + (40, 33) + ); + assert_eq!( + SampleAspectRatio::new(u32::MAX, u32::MAX) + .unwrap() + .as_ratio(), + (1, 1) + ); + let mut s = settings(1920, 1080, 26); + s.frame_rate = FrameRate::new(983040, 8160).unwrap(); + assert!(Encoder::new(s, Limits::default()).is_ok()); + s.frame_rate = FrameRate::new(983041, 8160).unwrap(); + assert!(matches!( + Encoder::new(s, Limits::default()), + Err(Error::Limit) + )); +} + +#[test] +fn vui_rationals_preserve_syntax_and_native_picture_roundtrips() { + let sars = [ + (1, 1), + (12, 11), + (10, 11), + (16, 11), + (40, 33), + (24, 11), + (20, 11), + (32, 11), + (80, 33), + (18, 11), + (15, 11), + (64, 33), + (160, 99), + (4, 3), + (3, 2), + (2, 1), + (17, 13), + (65535, 65534), + ]; + let output = std::env::var_os("H264_VUI_FIXTURE_DIR").map(std::path::PathBuf::from); + if let Some(path) = &output { + std::fs::create_dir_all(path).unwrap(); + } + let y = [128; 256]; + let uv = [128; 64]; + let picture = Picture { + y: Plane::new(&y, 16, 16, 16).unwrap(), + cb: Plane::new(&uv, 8, 8, 8).unwrap(), + cr: Plane::new(&uv, 8, 8, 8).unwrap(), + }; + for (num, den) in [ + (24000, 1001), + (25, 1), + (30000, 1001), + (50, 1), + (60, 1), + (30, 1), + (25, 2), + ] { + for (index, (width, height)) in sars.into_iter().enumerate() { + let mut s = settings(16, 16, 26); + s.frame_rate = FrameRate::new(num, den).unwrap(); + s.sample_aspect_ratio = SampleAspectRatio::new(width, height).unwrap(); + s.gop = Gop::Fixed { idr_interval: 32 }; + let mut encoder = Encoder::new(s, Limits::default()).unwrap(); + let mut decoder = coreav_h264::Decoder::with_kernels( + coreav_h264::Scalar, + coreav_h264::Limits::default(), + ) + .with_mode(coreav_h264::Mode::EveryPicture); + let mut stream = Vec::new(); + for _ in 0..64 { + let au = encoder.encode(picture).unwrap(); + let mut r = nal_reader(&au.nals[0]); + assert_eq!(r.u(24), 0x42c033); + for expected in [0, 0, 2, 1] { + assert_eq!(r.ue(), expected); + } + assert_eq!(r.bit(), 0); + assert_eq!(r.ue(), 0); + assert_eq!(r.ue(), 0); + assert_eq!(r.u(4), 0b1101); + assert_eq!(r.bit(), 1); + assert_eq!(r.u(8), if index < 16 { index as u32 + 1 } else { 255 }); + if index >= 16 { + assert_eq!(r.u(16), width); + assert_eq!(r.u(16), height); + } + assert_eq!(r.bit(), 0); + assert_eq!(r.bit(), 1); + assert_eq!(r.u(5), 0b10101); + assert_eq!(r.u(24), 0x010101); + assert_eq!(r.bit(), 1); + assert_eq!(r.ue(), 0); + assert_eq!(r.ue(), 0); + assert_eq!(r.bit(), 1); + assert_eq!(r.u(32), if den % 2 == 0 { den / 2 } else { den }); + assert_eq!(r.u(32), if den % 2 == 0 { num } else { 2 * num }); + assert_eq!(r.bit(), 1); + assert_eq!(r.u(4), 0); + for nal in &au.nals { + stream.extend_from_slice(&[0, 0, 0, 1]); + stream.extend_from_slice(nal); + decoder.decode_nal(nal).unwrap(); + } + decoder.end_access_unit().unwrap(); + } + decoder.flush().unwrap(); + let pictures = decoder.take_pictures(); + assert_eq!(pictures.len(), 64); + for picture in pictures { + let (cb, cr) = picture.chroma(); + assert!( + picture + .luma() + .rows() + .chain(cb.rows()) + .chain(cr.rows()) + .flatten() + .all(|&sample| sample == 128) + ); + } + if let Some(path) = &output { + std::fs::write( + path.join(format!("{num}-{den}-{width}-{height}.h264")), + stream, + ) + .unwrap(); + } + } + } +} + #[derive(Default)] struct Reader { bytes: Vec, diff --git a/crates/coreav-h264-encoder/tests/vui.py b/crates/coreav-h264-encoder/tests/vui.py new file mode 100644 index 000000000..cdccba617 --- /dev/null +++ b/crates/coreav-h264-encoder/tests/vui.py @@ -0,0 +1,78 @@ +"""Check generated encoder VUI with the qualified black-box ffprobe.""" +import argparse +from fractions import Fraction +import json +from pathlib import Path +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'scripts')) +import test_oracle +import h264_g1_conformance as syntax + + +def cadence(source): + r = syntax.rbsp(syntax.units(source.read_bytes())[0]) + syntax.require(r.u(24), 0x42c033, 'profile constraints level') + syntax.require([r.ue() for _ in range(4)], [0, 0, 2, 1], 'SPS fixed fields') + syntax.require(r.u(1), 0, 'frame number gaps') + syntax.require([r.ue(), r.ue()], [0, 0], '16x16 coded geometry') + syntax.require(r.u(4), 0b1101, 'progressive uncropped VUI') + syntax.require(r.u(1), 1, 'aspect ratio present') + if r.u(8) == 255: + r.u(32) + syntax.require(r.u(7), 0b0110101, 'overscan and video signal') + syntax.require(r.u(24), 0x010101, 'colour') + syntax.require(r.u(1), 1, 'chroma location present') + syntax.require([r.ue(), r.ue()], [0, 0], 'chroma location') + syntax.require(r.u(1), 1, 'timing present') + tick, scale = r.u(32), r.u(32) + syntax.require(r.u(1), 1, 'fixed frame rate') + if not tick or not scale: + raise ValueError('zero VUI clock') + return Fraction(scale, 2 * tick) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--ffmpeg', type=Path, required=True) + parser.add_argument('--fixtures', type=Path, required=True) + args = parser.parse_args() + oracle = test_oracle.require(args.ffmpeg) + rates = ((24000, 1001), (25, 1), (30000, 1001), (50, 1), (60, 1), (30, 1), (25, 2)) + sars = ((1, 1), (12, 11), (10, 11), (16, 11), (40, 33), (24, 11), + (20, 11), (32, 11), (80, 33), (18, 11), (15, 11), (64, 33), + (160, 99), (4, 3), (3, 2), (2, 1), (17, 13), (65535, 65534)) + count = 0 + for num, den in rates: + for width, height in sars: + source = args.fixtures / f'{num}-{den}-{width}-{height}.h264' + rate = cadence(source) + if rate != Fraction(num, den): + raise RuntimeError(f'{source.name}: SPS cadence={rate}') + prepared = source.with_suffix('.mp4') + subprocess.run([ + str(oracle.ffmpeg), '-v', 'error', '-r', str(rate), '-i', str(source), + '-c:v', 'copy', '-video_track_timescale', str(rate.numerator), + '-y', str(prepared), + ], check=True, capture_output=True, timeout=30) + result = subprocess.run([ + str(oracle.ffprobe), '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'stream=r_frame_rate,avg_frame_rate,sample_aspect_ratio', + '-of', 'json', str(prepared), + ], check=True, capture_output=True, text=True, timeout=30) + streams = json.loads(result.stdout)['streams'] + if len(streams) != 1: + raise RuntimeError(f'{source.name}: expected one video stream') + stream = streams[0] + for field in ('r_frame_rate', 'avg_frame_rate'): + if Fraction(stream[field]) != Fraction(num, den): + raise RuntimeError(f'{source.name}: {field}={stream[field]}, stream={stream}') + if Fraction(stream['sample_aspect_ratio'].replace(':', '/')) != Fraction(width, height): + raise RuntimeError(f'{source.name}: SAR={stream["sample_aspect_ratio"]}') + count += 1 + print(f'VUI oracle: {count} rate/SAR combinations passed') + + +if __name__ == '__main__': + main()