diff --git a/.github/workflows/hevc-reference.yml b/.github/workflows/hevc-reference.yml index 606fa0644..13f18b74d 100644 --- a/.github/workflows/hevc-reference.yml +++ b/.github/workflows/hevc-reference.yml @@ -1,5 +1,5 @@ name: hevc-reference -run-name: HEVC reference ${{ inputs.candidate }} +run-name: HEVC reference ${{ inputs.candidate }} (${{ inputs.only || 'all' }}, cap ${{ inputs.frames }}) # workflow-check: not required on: @@ -9,11 +9,26 @@ on: description: Exact decoder commit required: true type: string + baseline: + description: Optional full main SHA for the threading comparison + required: false + default: '' + type: string + sanitizer_only: + description: Run only the threading sanitizer stress check + required: false + default: false + type: boolean only: description: Optional comma-separated input IDs required: false default: '' type: string + frames: + description: Maximum pictures per reference input + required: false + default: 32 + type: number diagnose_only: description: Compare pixels without timing or profiling required: false @@ -25,12 +40,14 @@ permissions: jobs: reference: + if: inputs.baseline == '' && !inputs.sanitizer_only runs-on: ubuntu-24.04 timeout-minutes: 150 env: CANDIDATE: ${{ inputs.candidate }} ONLY: ${{ inputs.only }} DIAGNOSE_ONLY: ${{ inputs.diagnose_only }} + FRAMES: ${{ inputs.frames }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: @@ -81,7 +98,7 @@ jobs: python3 -I driver/scripts/hevc_reference.py \ --native "$PWD/candidate/target/release/examples/reference" \ --oracle-bin "$RUNNER_TEMP/oracle" --out "$RUNNER_TEMP/reference" \ - --candidate "$CANDIDATE" --only "$ONLY" "${EXTRA[@]}" + --candidate "$CANDIDATE" --only "$ONLY" --frames "$FRAMES" "${EXTRA[@]}" - name: Bind retained results to the dispatched decoder run: | python3 -I driver/scripts/hevc_reference.py check-result \ @@ -95,3 +112,14 @@ jobs: ${{ runner.temp }}/compiler.txt if-no-files-found: error retention-days: 30 + + threading: + if: inputs.baseline != '' || inputs.sanitizer_only + uses: ./.github/workflows/hevc-threading.yml + with: + candidate: ${{ inputs.candidate }} + baseline: ${{ inputs.baseline }} + sanitizer_only: ${{ inputs.sanitizer_only }} + only: ${{ inputs.only }} + frames: ${{ fromJSON(inputs.frames) }} + secrets: inherit diff --git a/.github/workflows/hevc-threading.yml b/.github/workflows/hevc-threading.yml new file mode 100644 index 000000000..db7b0b910 --- /dev/null +++ b/.github/workflows/hevc-threading.yml @@ -0,0 +1,197 @@ +name: hevc-threading +# workflow-check: not required + +on: + workflow_call: + inputs: + candidate: + required: true + type: string + baseline: + required: true + type: string + sanitizer_only: + required: false + default: false + type: boolean + only: + required: false + default: '' + type: string + frames: + required: false + default: 32 + type: number + +permissions: + contents: read + +env: + CANDIDATE: ${{ inputs.candidate }} + BASELINE: ${{ inputs.baseline }} + ONLY: ${{ inputs.only }} + FRAMES: ${{ inputs.frames }} + +jobs: + reference: + if: '!inputs.sanitizer_only' + name: hevc-threading-reference (${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(inputs.only != '' && '[0]' || '[0,1,2]') }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.candidate }} + fetch-depth: 0 + path: candidate + - name: Admit exact branch arms before building or using credentials + working-directory: candidate + run: | + set -euo pipefail + [[ "$CANDIDATE" =~ ^[0-9a-f]{40}$ && "$BASELINE" =~ ^[0-9a-f]{40}$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -n "$(git branch -r --contains "$CANDIDATE")" + git merge-base --is-ancestor "$BASELINE" origin/main + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.baseline }} + path: baseline + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + with: + toolchain: stable + - name: Build distinct candidate and serial-main executables + run: | + set -euo pipefail + cp candidate/bench/hevc-reference/decode.rs candidate/crates/coreav-hevc/examples/reference.rs + cp candidate/bench/hevc-reference/decode.rs baseline/crates/coreav-hevc/examples/reference.rs + (cd candidate && RUSTFLAGS='--check-cfg=cfg(hevc_serial_baseline)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference --example headers) + (cd baseline && test "$(git rev-parse HEAD)" = "$BASELINE" && RUSTFLAGS='--cfg hevc_serial_baseline --check-cfg=cfg(hevc_serial_baseline)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference) + - name: Install the pinned binary oracle + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: candidate/scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/oracle" + - name: Verify every thread count before paired reference calls + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: | + umask 077 + mkdir -p "$RUNNER_TEMP/private" + python3 -I candidate/scripts/hevc_threading.py \ + --native "$PWD/candidate/target/release/examples/reference" \ + --main "$PWD/baseline/target/release/examples/reference" \ + --oracle-bin "$RUNNER_TEMP/oracle" --out "$RUNNER_TEMP/private" \ + --candidate "$CANDIDATE" --baseline "$BASELINE" \ + --shard-index '${{ matrix.shard }}' --shard-count '${{ inputs.only != '' && 1 || 3 }}' --only "$ONLY" --frames "$FRAMES" + - name: Seal reference results + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + [ -d private ] || exit 0 + tar -czf records.tgz private + mkdir -p sealed + openssl rand -hex -out key.txt 32 + openssl enc -aes-256-cbc -pbkdf2 -salt -pass file:key.txt -in records.tgz -out sealed/records.tgz.enc + openssl pkeyutl -encrypt -pubin -inkey "$GITHUB_WORKSPACE/candidate/bench/hevc-conformance/evidence-public.pem" -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -in key.txt -out sealed/key.txt.enc + shred -u key.txt records.tgz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: hevc-threading-reference-${{ matrix.shard }}-${{ inputs.candidate }} + path: ${{ runner.temp }}/sealed + if-no-files-found: error + retention-days: 7 + + conformance: + if: ${{ !inputs.sanitizer_only && inputs.only == '' }} + name: hevc-threading-conformance + runs-on: ubuntu-24.04 + timeout-minutes: 360 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.candidate }} + fetch-depth: 0 + - name: Admit the exact candidate + run: | + set -euo pipefail + [[ "$CANDIDATE" =~ ^[0-9a-f]{40}$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -n "$(git branch -r --contains "$CANDIDATE")" + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + with: + toolchain: stable + - run: cargo build --locked --release -p coreav-hevc --features native-hevc --example headers --example decode + - run: scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/oracle" + - name: Prepare complete conformance references + run: | + set -euo pipefail + umask 077 + mkdir -p "$RUNNER_TEMP/private" + python3 -I scripts/hevc_fixture.py verify --ffmpeg "$RUNNER_TEMP/oracle/ffmpeg" --ffprobe "$RUNNER_TEMP/oracle/ffprobe" --output "$RUNNER_TEMP/synthetic" > "$RUNNER_TEMP/private/synthetic.log" 2>&1 + python3 -I scripts/hevc_conformance.py acquire --dest "$RUNNER_TEMP/data" > "$RUNNER_TEMP/private/acquire.log" 2>&1 + python3 -I scripts/hevc_conformance.py manifest --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/archives.json" > "$RUNNER_TEMP/private/manifest.log" 2>&1 + python3 -I scripts/hevc_conformance.py run --registration "$RUNNER_TEMP/private/archives.json" --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/census" --parser "$PWD/target/release/examples/headers" --oracle-bin "$RUNNER_TEMP/oracle" --synthetic-data "$RUNNER_TEMP/synthetic" > "$RUNNER_TEMP/private/census.log" 2>&1 + - name: Compare complete outputs at one two four and eight threads + run: | + set -euo pipefail + for threads in 1 2 4 8; do + python3 -I scripts/hevc_decode.py --registration "$RUNNER_TEMP/private/archives.json" --census "$RUNNER_TEMP/private/census/census.json" --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/threads-$threads" --parser "$PWD/target/release/examples/headers" --native "$PWD/target/release/examples/decode" --oracle-bin "$RUNNER_TEMP/oracle" --threads "$threads" > "$RUNNER_TEMP/private/threads-$threads.log" 2>&1 + done + - name: Seal conformance records + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + [ -d private ] || exit 0 + tar -czf records.tgz private + mkdir -p sealed + openssl rand -hex -out key.txt 32 + openssl enc -aes-256-cbc -pbkdf2 -salt -pass file:key.txt -in records.tgz -out sealed/records.tgz.enc + openssl pkeyutl -encrypt -pubin -inkey "$GITHUB_WORKSPACE/bench/hevc-conformance/evidence-public.pem" -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -in key.txt -out sealed/key.txt.enc + shred -u key.txt records.tgz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: hevc-threading-conformance-${{ inputs.candidate }} + path: ${{ runner.temp }}/sealed + if-no-files-found: error + retention-days: 7 + + thread-sanitizer: + if: ${{ inputs.sanitizer_only || inputs.only == '' }} + name: hevc-threading-sanitizer + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.candidate }} + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + with: + toolchain: nightly + components: rust-src + - name: Stress wavefront completion and failure under ThreadSanitizer + env: + RUSTFLAGS: -Zsanitizer=thread + RUSTDOCFLAGS: -Zsanitizer=thread + run: | + set -euo pipefail + sudo sysctl vm.mmap_rnd_bits=28 + for repeat in $(seq 1 20); do + cargo +nightly test --locked -Zbuild-std --target x86_64-unknown-linux-gnu -p coreav-hevc --features native-hevc --lib wavefront + done diff --git a/bench/hevc-conformance/README.md b/bench/hevc-conformance/README.md index f62aeea35..fb766b5a4 100644 --- a/bench/hevc-conformance/README.md +++ b/bench/hevc-conformance/README.md @@ -53,7 +53,7 @@ python3 -I scripts/hevc_decode.py \ --max-picture-bins 268435456 ``` -The comparator consumes the complete source-bound census, preserves every archive disposition and requires the published whole-output anchor before comparing individual output pictures. It checks original coded/output identities, geometry, packing, hashes, effective work and budget controls. `--select-test` is diagnostic only and cannot report complete coverage. The `decode` example accepts `INPUT MAX_MEMORY_BYTES MAX_STREAM_BINS MAX_PICTURE_BINS`; `-` reads standard input. Its bounded JSON lines contain each output picture and a final successful-completion summary. Annex B timestamps remain null. This command and its fixture tests are implemented. Original-media C1 and C2 comparisons run in the full tier, not the routine smoke. +The comparator consumes the complete source-bound census, preserves every archive disposition and requires the published whole-output anchor before comparing individual output pictures. It checks original coded/output identities, geometry, packing, hashes, effective work and budget controls. `--select-test` is diagnostic only and cannot report complete coverage. The `decode` example accepts `INPUT MAX_MEMORY_BYTES MAX_STREAM_BINS MAX_PICTURE_BINS [THREADS]`; `-` reads standard input. The optional `hevc_decode.py --threads N` supplies that last argument to both valid-stream comparisons and failure controls. Without it, decoding uses one worker. Its bounded JSON lines contain each output picture and a final successful-completion summary, including the actual maximum worker count and parallel-picture count. Annex B timestamps remain null. This command and its fixture tests are implemented. Original-media C1 and C2 comparisons run in the full tier, not the routine smoke. ### Container packet comparison diff --git a/bench/hevc-reference/decode.rs b/bench/hevc-reference/decode.rs index 4a1da938a..a7a47cd9c 100644 --- a/bench/hevc-reference/decode.rs +++ b/bench/hevc-reference/decode.rs @@ -16,6 +16,24 @@ fn run(writer: &mut impl Write) -> Result<()> { let mode = args .next() .ok_or(Error::InvalidData("reference_arguments"))?; + let threads = args + .next() + .map(|value| { + value + .into_string() + .ok() + .and_then(|value| value.parse::().ok()) + }) + .unwrap_or(Some(1)) + .filter(|value| *value <= 16) + .ok_or(Error::InvalidData("reference_arguments"))?; + let threads = if threads == 0 { + std::thread::available_parallelism() + .map_or(1, usize::from) + .min(16) + } else { + threads + }; if args.next().is_some() || (mode != "check" && mode != "decode") { return Err(Error::InvalidData("reference_arguments")); } @@ -29,10 +47,19 @@ fn run(writer: &mut impl Write) -> Result<()> { return Err(Error::Limit("reference_input")); } let limits = DecodeLimits { - max_memory_bytes: 1 << 30, + max_memory_bytes: usize::try_from(coreav_decode::Limits::default().max_memory_bytes) + .map_err(|_| Error::Limit("reference_memory"))?, ..DecodeLimits::default() }; let mut decoder = PictureDecoder::new(limits); + #[cfg(not(hevc_serial_baseline))] + { + decoder = decoder.with_threads(threads); + } + #[cfg(hevc_serial_baseline)] + { + std::hint::black_box(threads); + } let mut output = decoded::Output::new(writer); let mut pictures = 0u64; let mut callback = |picture: coreav_hevc::DecodedPicture<'_>| { @@ -51,12 +78,15 @@ fn run(writer: &mut impl Write) -> Result<()> { if mode == "check" { output.finish(summary, receipt, limits) } else { - serde_json::to_writer( - &mut *writer, - &json!({"pictures":pictures, - "decoded_pictures":summary.decoded_pictures,"bins":summary.reconstruction.bins}), - ) - .map_err(|_| Error::Limit("reference_output"))?; + let mut record = json!({"pictures":pictures,"thread_limit":threads,"max_memory_bytes":limits.max_memory_bytes, + "decoded_pictures":summary.decoded_pictures,"bins":summary.reconstruction.bins}); + #[cfg(not(hevc_serial_baseline))] + { + record["parallel_pictures"] = json!(summary.parallel_pictures); + record["max_workers"] = json!(summary.max_workers); + } + serde_json::to_writer(&mut *writer, &record) + .map_err(|_| Error::Limit("reference_output"))?; writer .write_all(b"\n") .map_err(|_| Error::Limit("reference_output")) diff --git a/crates/coreav-backend-apple/src/hevc.rs b/crates/coreav-backend-apple/src/hevc.rs index 23ad4b933..68d557dd3 100644 --- a/crates/coreav-backend-apple/src/hevc.rs +++ b/crates/coreav-backend-apple/src/hevc.rs @@ -9,6 +9,8 @@ use coreav_hevc::{HeaderParser, LengthPrefixed, NalHeader, NalInfo}; /// A refusal of the checked HEVC output contract. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { + /// An internal decoder or validator failure, not invalid output metadata. + Internal, /// Truncated or invalid syntax or framing. Malformed, /// A profile, coding tool, or NAL type outside the output contract. @@ -32,6 +34,7 @@ pub enum Error { impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { + Self::Internal => "internal HEVC failure", Self::Malformed => "malformed HEVC output metadata", Self::Unsupported => "unsupported HEVC output metadata", Self::FormatMismatch => "HEVC output format changed", @@ -50,6 +53,7 @@ impl std::error::Error for Error {} impl From for Error { fn from(error: coreav_hevc::Error) -> Self { match error { + coreav_hevc::Error::Internal(_) => Self::Internal, coreav_hevc::Error::InvalidData(_) => Self::Malformed, coreav_hevc::Error::Unsupported(_) => Self::Unsupported, coreav_hevc::Error::Limit(_) => Self::Limit, diff --git a/crates/coreav-backend-apple/tests/hevc_output.rs b/crates/coreav-backend-apple/tests/hevc_output.rs index f86e1ccd0..50720aa6d 100644 --- a/crates/coreav-backend-apple/tests/hevc_output.rs +++ b/crates/coreav-backend-apple/tests/hevc_output.rs @@ -5,6 +5,14 @@ use coreav_backend_apple::{ hevc::{AccessUnit, Config, Error, Stream}, }; +#[test] +fn decoder_internal_failure_is_not_a_metadata_or_resource_refusal() { + assert_eq!( + Error::from(coreav_hevc::Error::Internal("reconstruction_panic")), + Error::Internal + ); +} + #[derive(Default)] struct Writer(Vec); diff --git a/crates/coreav-backend-hevc/src/lib.rs b/crates/coreav-backend-hevc/src/lib.rs index ac9581198..d9a01085c 100644 --- a/crates/coreav-backend-hevc/src/lib.rs +++ b/crates/coreav-backend-hevc/src/lib.rs @@ -23,12 +23,27 @@ const DECODER_NAME: &str = "coreav hevc, scalar Main/Main10"; /// Scalar native eight-bit Main and eight- or ten-bit Main10 decoding. #[derive(Clone, Copy, Debug, Default)] -pub struct NativeHevc; +pub struct NativeHevc { + options: NativeOptions, +} + +/// Reconstruction options for the opt-in native decoder. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct NativeOptions { + /// Reconstruction workers; `None` selects available host parallelism. + /// Counts are bounded to one through sixteen and reduced to fit memory. + pub threads: Option, +} impl NativeHevc { /// Creates the opt-in backend without activating any decoder session. pub fn new() -> Self { - Self + Self::default() + } + + /// Creates a backend with caller-selected reconstruction options. + pub fn with_options(options: NativeOptions) -> Self { + Self { options } } } @@ -59,6 +74,10 @@ impl Backend for NativeHevc { } let defaults = coreav_hevc::Limits::default(); let deadline = limits.deadline(); + let threads = self + .options + .threads + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)); let mut decoder = PictureDecoder::new(DecodeLimits { headers: coreav_hevc::Limits { max_width: limits.max_width.min(defaults.max_width), @@ -76,7 +95,8 @@ impl Backend for NativeHevc { max_frame_bytes: limits.max_frame_bytes, deadline: Some(deadline), ..DecodeLimits::default() - }); + }) + .with_threads(threads); if (core::mem::size_of::() as u64).saturating_add(DECODER_NAME.len() as u64) > limits.max_memory_bytes { @@ -320,6 +340,7 @@ fn check_deadline(deadline: Instant) -> Result<(), Error> { fn codec_error(error: coreav_hevc::Error) -> Error { let kind = match error { + coreav_hevc::Error::Internal(_) => ErrorKind::Backend, coreav_hevc::Error::Limit("decode_deadline") => ErrorKind::Timeout, coreav_hevc::Error::Limit(_) => ErrorKind::Limit, coreav_hevc::Error::Unsupported(_) => ErrorKind::Unsupported, diff --git a/crates/coreav-backend-hevc/src/tests.rs b/crates/coreav-backend-hevc/src/tests.rs index 7a8a02338..1f5563a04 100644 --- a/crates/coreav-backend-hevc/src/tests.rs +++ b/crates/coreav-backend-hevc/src/tests.rs @@ -4,6 +4,14 @@ use coreav_frame::{ColorInfo, Pixels}; type Result = std::result::Result>; +#[test] +fn reconstruction_panics_are_backend_failures_not_media_limits() { + assert_eq!( + codec_error(coreav_hevc::Error::Internal("reconstruction_panic")).kind, + ErrorKind::Backend + ); +} + #[path = "../../coreav-hevc/test-support/pcm.rs"] mod pcm; #[path = "../../coreav-hevc/test-support/writer.rs"] diff --git a/crates/coreav-cli/src/thumbnail.rs b/crates/coreav-cli/src/thumbnail.rs index 97a5396fc..3fc623917 100644 --- a/crates/coreav-cli/src/thumbnail.rs +++ b/crates/coreav-cli/src/thumbnail.rs @@ -248,7 +248,11 @@ pub(crate) fn backends( #[cfg(feature = "native-vp9")] out.push(Box::new(coreav_backend_vp9::NativeVp9::new())); #[cfg(feature = "native-hevc")] - out.push(Box::new(coreav_backend_hevc::NativeHevc::new())); + out.push(Box::new(coreav_backend_hevc::NativeHevc::with_options( + coreav_backend_hevc::NativeOptions { + threads: (threads != 0).then_some(threads as usize), + }, + ))); #[cfg(any(feature = "ffmpeg-fallback", feature = "dav1d", feature = "libvpx"))] if let Some(program) = this { use coreav_worker::{SandboxedBackend, WorkerBackend, WorkerCommand, WorkerOptions}; diff --git a/crates/coreav-hevc/examples/common/decoded.rs b/crates/coreav-hevc/examples/common/decoded.rs index ac502991e..60b40a1df 100644 --- a/crates/coreav-hevc/examples/common/decoded.rs +++ b/crates/coreav-hevc/examples/common/decoded.rs @@ -215,6 +215,7 @@ impl<'a, W: Write> Output<'a, W> { "independent_slices":headers.independent_slices,"dependent_slices":headers.dependent_slices, "picture_hashes":headers.picture_hashes,"reserved_hashes":headers.reserved_hashes}, "decoded_pictures":summary.decoded_pictures,"skipped_pictures":summary.skipped_pictures, + "parallel_pictures":summary.parallel_pictures,"max_workers":summary.max_workers, "skipped_picture_hashes":summary.skipped_picture_hashes,"skipped_reserved_hashes":summary.skipped_reserved_hashes, "stats":common::reconstruction_stats(summary.reconstruction),"inter":inter_stats(summary.inter), "memory_bytes":summary.memory_bytes,"input_bytes":input.bytes,"input_capacity":input.capacity, diff --git a/crates/coreav-hevc/examples/decode.rs b/crates/coreav-hevc/examples/decode.rs index cb90c2256..c5757c3e8 100644 --- a/crates/coreav-hevc/examples/decode.rs +++ b/crates/coreav-hevc/examples/decode.rs @@ -42,6 +42,11 @@ fn run(writer: &mut impl Write) -> Result<()> { max_memory_bytes: number(args.next())?, max_stream_bins: number(args.next())?, max_picture_bins: number(args.next())?, + threads: args + .next() + .map(|value| number(Some(value))) + .transpose()? + .unwrap_or(1), ..DecodeLimits::default() }; if args.next().is_some() { @@ -64,6 +69,7 @@ fn main() -> std::process::ExitCode { Error::InvalidData(reason) => ("invalid_data", reason), Error::Unsupported(reason) => ("unsupported", reason), Error::Limit(reason) => ("limit", reason), + Error::Internal(reason) => ("backend", reason), }; let mut stderr = std::io::stderr().lock(); let _ = serde_json::to_writer( @@ -401,6 +407,8 @@ mod tests { fn summary(count: u64) -> DecodeSummary { DecodeSummary { + parallel_pictures: 0, + max_workers: 1, headers: Summary { warnings: coreav_hevc::Warnings { profile_bit_depth: true, diff --git a/crates/coreav-hevc/examples/headers.rs b/crates/coreav-hevc/examples/headers.rs index 5d7eeae03..59401ddde 100644 --- a/crates/coreav-hevc/examples/headers.rs +++ b/crates/coreav-hevc/examples/headers.rs @@ -189,6 +189,7 @@ fn main() -> std::process::ExitCode { Error::InvalidData(reason) => ("invalid_data", reason), Error::Unsupported(reason) => ("unsupported", reason), Error::Limit(reason) => ("limit", reason), + Error::Internal(reason) => ("backend", reason), }; let _ = row( &mut output, diff --git a/crates/coreav-hevc/examples/intra.rs b/crates/coreav-hevc/examples/intra.rs index c60b9e5ac..f2fecd58d 100644 --- a/crates/coreav-hevc/examples/intra.rs +++ b/crates/coreav-hevc/examples/intra.rs @@ -172,6 +172,7 @@ fn main() -> std::process::ExitCode { Error::InvalidData(reason) => ("invalid_data", reason), Error::Unsupported(reason) => ("unsupported", reason), Error::Limit(reason) => ("limit", reason), + Error::Internal(reason) => ("backend", reason), }; let mut stderr = std::io::stderr().lock(); let _ = serde_json::to_writer( diff --git a/crates/coreav-hevc/examples/packets.rs b/crates/coreav-hevc/examples/packets.rs index 65a0e4b1d..59c51d521 100644 --- a/crates/coreav-hevc/examples/packets.rs +++ b/crates/coreav-hevc/examples/packets.rs @@ -398,6 +398,7 @@ fn main() -> std::process::ExitCode { Error::InvalidData(reason) => ("invalid_data", reason), Error::Unsupported(reason) => ("unsupported", reason), Error::Limit(reason) => ("limit", reason), + Error::Internal(reason) => ("backend", reason), }; let mut stderr = std::io::stderr().lock(); let _ = serde_json::to_writer( diff --git a/crates/coreav-hevc/src/cabac.rs b/crates/coreav-hevc/src/cabac.rs index d0137f859..a1a5f5c96 100644 --- a/crates/coreav-hevc/src/cabac.rs +++ b/crates/coreav-hevc/src/cabac.rs @@ -62,6 +62,13 @@ pub(crate) struct Cabac<'a, 'b> { } impl<'a, 'b> Cabac<'a, 'b> { + pub(crate) fn max_bins(bytes: usize) -> u64 { + (bytes as u64) + .saturating_mul(8) + .saturating_add(1) + .saturating_mul(128) + } + pub(crate) fn new(bytes: &'a [u8], remaining_bin_budget: &'b mut u64) -> Result { let mut decoder = Self { input: Bits::new(bytes), @@ -312,6 +319,36 @@ mod tests { use super::{Cabac, Context, Initialization}; use crate::Error; + #[test] + fn substream_bound_covers_the_longest_mps_runs_and_truncated_reads() { + assert!( + super::RANGE_TAB_LPS + .iter() + .flatten() + .all(|range| *range >= 2) + ); + for length in 2..=64 { + let bytes = vec![0; length]; + let allowance = Cabac::max_bins(length); + let mut remaining = allowance; + let mut decoder = Cabac::new(&bytes, &mut remaining).unwrap(); + let mut context = Context { + state: 63, + mps: false, + }; + loop { + match decoder.decision(&mut context) { + Ok(_) => {} + Err(error) => { + assert_eq!(error, Error::InvalidData("cabac_truncated")); + break; + } + } + } + assert!(remaining > 0); + } + } + #[test] fn slice_initialization_and_swap_follow_equation_9_7() { for (kind, swap, expected) in [ diff --git a/crates/coreav-hevc/src/decoder.rs b/crates/coreav-hevc/src/decoder.rs index c976f2b00..780bce084 100644 --- a/crates/coreav-hevc/src/decoder.rs +++ b/crates/coreav-hevc/src/decoder.rs @@ -2,12 +2,14 @@ use crate::{ Error, InterStats, IntraPlane, IntraStats, Limits, PictureInfo, Result, SeiInfo, Summary, dpb::{Dpb, Emission}, parser::{PictureObserver, SliceInput, StreamCore}, - reconstruction::{Frame, Memory}, + reconstruction::{Cache, Frame, Memory, ReclaimMemory}, }; /// Independent admission, live-memory and entropy-work limits for a complete stream. #[derive(Clone, Copy, Debug)] pub struct DecodeLimits { + /// Requested reconstruction workers, bounded to one through sixteen. + pub threads: usize, /// Header admission, including the stream-wide coded-picture limit. pub headers: Limits, /// Maximum simultaneous codec state, pictures and transient scratch bytes. @@ -25,6 +27,7 @@ pub struct DecodeLimits { impl Default for DecodeLimits { fn default() -> Self { Self { + threads: 1, headers: Limits::default(), max_memory_bytes: 512 * 1024 * 1024, max_stream_bins: 1 << 40, @@ -61,6 +64,7 @@ pub struct OutputMemory<'a> { codec_bytes: usize, reserved: &'a mut usize, limit: usize, + reclaim: Option<&'a mut dyn ReclaimMemory>, } impl OutputMemory<'_> { @@ -70,7 +74,14 @@ impl OutputMemory<'_> { .reserved .checked_add(bytes) .ok_or(Error::Limit("reconstruction_memory"))?; - let mut memory = Memory::new(self.codec_bytes, self.limit)?; + let mut memory = Memory::with_reclaim( + self.codec_bytes, + self.limit, + match &mut self.reclaim { + Some(reclaim) => Some(&mut **reclaim), + None => None, + }, + )?; memory.charge(reserved)?; *self.reserved = reserved; Ok(()) @@ -89,6 +100,10 @@ impl OutputMemory<'_> { /// Successful complete-stream accounting, including pictures not selected for output. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct DecodeSummary { + /// Pictures reconstructed using more than one successfully started worker. + pub parallel_pictures: u64, + /// Greatest number of successfully started workers, including the caller. + pub max_workers: usize, /// Header syntax and presentation-order output counts. pub headers: Summary, /// Pictures reconstructed and finalized, including non-output reference pictures. @@ -115,9 +130,44 @@ enum State { Failed, } -struct Pictures { +#[derive(Default)] +struct Slots { + active: Vec, + cache: Cache, +} + +impl Slots { + fn as_ref(&self) -> Option<&Frame> { + self.active.first() + } + fn is_none(&self) -> bool { + self.active.is_empty() + } + fn take(&mut self) -> Option { + self.active.pop() + } + + fn prepare(&mut self) -> Result<()> { + if self.active.capacity() == 0 { + self.active + .try_reserve_exact(1) + .map_err(|_| Error::Limit("picture_allocation"))?; + } + Ok(()) + } + + fn put(&mut self, frame: Frame) -> Result<()> { + if !self.active.is_empty() || self.active.capacity() == 0 { + return Err(Error::Internal("working_picture_storage")); + } + self.active.push(frame); + Ok(()) + } +} + +struct Pictures { limits: DecodeLimits, - frame: Option, + frame: Storage, timestamp: Option, origin: Option, external_memory: usize, @@ -127,9 +177,9 @@ struct Pictures { summary: DecodeSummary, } -struct Session { +struct Session { core: StreamCore, - pictures: Pictures, + pictures: Pictures, } /// Sans-IO Main/Main10 scalar decoder with one DPB and borrowed output callbacks. @@ -140,6 +190,16 @@ pub struct PictureDecoder { } impl PictureDecoder { + /// Selects reconstruction workers without changing sample or output ordering. + pub fn with_threads(mut self, threads: usize) -> Self { + self.limits.threads = threads.clamp(1, 16); + for session in &mut self.session { + session.pictures.limits.threads = self.limits.threads; + session.pictures.frame.cache.clear(); + } + self + } + /// Creates an opt-in decoder without allocating picture samples. pub fn new(limits: DecodeLimits) -> Self { Self { @@ -152,7 +212,7 @@ impl PictureDecoder { fn fixed_memory(&self) -> Result { core::mem::size_of::() .checked_add( - core::mem::size_of::() + core::mem::size_of::>>() .checked_mul(self.session.capacity().max(1) + 1) .ok_or(Error::Limit("reconstruction_memory"))?, ) @@ -181,7 +241,7 @@ impl PictureDecoder { core: StreamCore::new_reconstruction(limits.headers), pictures: Pictures { limits, - frame: None, + frame: Slots::default(), timestamp: None, origin: None, external_memory: 0, @@ -248,10 +308,14 @@ impl PictureDecoder { .and_then(|n| n.checked_add(frame_bytes)) .and_then(|n| n.checked_add(session.pictures.external_memory)) .ok_or(Error::Limit("reconstruction_memory"))?; - let mut memory = Memory::new(live, session.pictures.limits.max_memory_bytes)?; + let mut memory = Memory::with_reclaim( + live, + session.pictures.limits.max_memory_bytes, + Some(&mut session.pictures.frame.cache), + )?; memory.charge(bytes.len().saturating_sub(2))?; session.pictures.summary.memory_bytes = - session.pictures.summary.memory_bytes.max(memory.used); + session.pictures.summary.memory_bytes.max(memory.peak()); let available = session .pictures .limits @@ -269,6 +333,11 @@ impl PictureDecoder { emit: &mut emit, }, )?; + session.pictures.summary.memory_bytes = session + .pictures + .summary + .memory_bytes + .max(session.pictures.frame.cache.peak()); Ok(()) })(); if result.is_err() { @@ -308,6 +377,12 @@ impl PictureDecoder { emit: &mut emit, })?; session.pictures.summary.headers = headers; + session.pictures.summary.memory_bytes = session + .pictures + .summary + .memory_bytes + .max(session.pictures.frame.cache.peak()); + session.pictures.frame.cache.clear(); Ok(session.pictures.summary) })(); if result.is_err() { @@ -340,11 +415,15 @@ impl PictureDecoder { .checked_add(session.core.heap_bytes()?) .and_then(|n| n.checked_add(frame)) .ok_or(Error::Limit("reconstruction_memory"))?; - let mut memory = Memory::new(live, self.limits.max_memory_bytes)?; + let mut memory = Memory::with_reclaim( + live, + self.limits.max_memory_bytes, + Some(&mut session.pictures.frame.cache), + )?; memory.charge(bytes)?; session.pictures.external_memory = bytes; session.pictures.summary.memory_bytes = - session.pictures.summary.memory_bytes.max(memory.used); + session.pictures.summary.memory_bytes.max(memory.peak()); Ok(()) })(); if result.is_err() { @@ -365,6 +444,28 @@ struct Observer<'a, F> { impl, &mut OutputMemory<'_>) -> Result<()>> PictureObserver for Observer<'_, F> { + fn reserve_scratch(&mut self, scratch_bytes: usize, dpb: &Dpb) -> Result<()> { + let frame = self + .pictures + .frame + .as_ref() + .map_or(Ok(0), Frame::heap_bytes)?; + let fixed = self + .fixed + .checked_add(dpb.heap_bytes()?) + .and_then(|n| n.checked_add(frame)) + .and_then(|n| n.checked_add(self.pictures.external_memory)) + .ok_or(Error::Limit("reconstruction_memory"))?; + let mut memory = Memory::with_reclaim( + fixed, + self.pictures.limits.max_memory_bytes, + Some(&mut self.pictures.frame.cache), + )?; + memory.charge(scratch_bytes)?; + self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(memory.peak()); + Ok(()) + } + fn boundary(&mut self, dpb: &mut Dpb, scratch_bytes: usize) -> Result<()> { self.pictures.skipped = None; let Some(mut frame) = self.pictures.frame.take() else { @@ -376,8 +477,19 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO .and_then(|n| n.checked_add(scratch_bytes)) .and_then(|n| n.checked_add(self.pictures.external_memory)) .ok_or(Error::Limit("reconstruction_memory"))?; - let peak = frame.finalize(fixed, self.pictures.limits.max_memory_bytes)?; - self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(peak); + let peak = frame.finalize_reclaiming( + fixed, + self.pictures.limits.max_memory_bytes, + Some(&mut self.pictures.frame.cache), + )?; + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(peak) + .max(self.pictures.frame.cache.peak()); + self.pictures.summary.parallel_pictures += u64::from(frame.workers > 1); + self.pictures.summary.max_workers = self.pictures.summary.max_workers.max(frame.workers); frame.stats.bins = self.pictures.limits.max_picture_bins - self.pictures.picture_remaining; frame.stats.memory_bytes = self.pictures.summary.memory_bytes; self.pictures @@ -431,40 +543,50 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO return Err(Error::Limit("output_frame_bytes")); } let identity = dpb.current_identity(input.index)?; + self.pictures.frame.prepare()?; let mut frame = Frame::new_full( &input, identity.poc, dpb.current_references(), fixed, self.pictures.limits.max_memory_bytes, + Some(&mut self.pictures.frame.cache), )?; frame.deadline = self.pictures.limits.deadline; - self.pictures.frame = Some(frame); + self.pictures.frame.put(frame)?; self.pictures.picture_remaining = self.pictures.limits.max_picture_bins; } - let frame = self - .pictures - .frame - .as_mut() + let Slots { active, cache } = &mut self.pictures.frame; + let frame = active + .first_mut() .ok_or(Error::InvalidData("missing_working_picture"))?; let allowance = self .pictures .stream_remaining .min(self.pictures.picture_remaining); let mut budget = allowance; - let result = frame.decode_full( - input, - dpb, - &mut budget, - fixed, - self.pictures.limits.max_memory_bytes, - ); + let result = crate::reconstruction::caught(|| { + frame.decode_full( + input, + dpb, + &mut budget, + fixed, + self.pictures.limits.max_memory_bytes, + self.pictures.limits.threads, + Some(cache), + ) + }); let consumed = allowance - budget; self.pictures.stream_remaining -= consumed; self.pictures.picture_remaining -= consumed; match result { Ok(peak) => { - self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(peak); + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(peak) + .max(self.pictures.frame.cache.peak()); Ok(()) } Err(Error::Limit("cabac_bin_budget")) if self.pictures.stream_remaining == 0 => { @@ -475,15 +597,26 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO } fn sei(&mut self, info: SeiInfo, dpb: &Dpb, scratch_bytes: usize) -> Result<()> { - if let Some(frame) = self.pictures.frame.as_mut() { + let Slots { active, cache } = &mut self.pictures.frame; + if let Some(frame) = active.first_mut() { let fixed = self .fixed .checked_add(dpb.heap_bytes()?) .and_then(|n| n.checked_add(scratch_bytes)) .and_then(|n| n.checked_add(self.pictures.external_memory)) .ok_or(Error::Limit("reconstruction_memory"))?; - let peak = frame.hash(info, fixed, self.pictures.limits.max_memory_bytes)?; - self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(peak); + let peak = frame.hash_reclaiming( + info, + fixed, + self.pictures.limits.max_memory_bytes, + Some(cache), + )?; + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(peak) + .max(cache.peak()); } else if self.pictures.skipped.is_some() { self.pictures.summary.skipped_picture_hashes = self .pictures @@ -523,6 +656,7 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO codec_bytes, reserved: &mut self.pictures.external_memory, limit: self.pictures.limits.max_memory_bytes, + reclaim: Some(&mut self.pictures.frame.cache), }; (self.emit)( DecodedPicture { @@ -537,15 +671,24 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO }, &mut memory, )?; - let mut used = Memory::new(codec_bytes, self.pictures.limits.max_memory_bytes)?; + let mut used = Memory::with_reclaim( + codec_bytes, + self.pictures.limits.max_memory_bytes, + Some(&mut self.pictures.frame.cache), + )?; used.charge(self.pictures.external_memory)?; - self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(used.used); + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(used.peak()) + .max(self.pictures.frame.cache.peak()); Ok(()) } } impl DecodeSummary { - fn add(&mut self, reconstruction: IntraStats, inter: InterStats) -> Result<()> { + pub(crate) fn add(&mut self, reconstruction: IntraStats, inter: InterStats) -> Result<()> { self.decoded_pictures = self .decoded_pictures .checked_add(1) @@ -664,6 +807,72 @@ fn add_counts(target: &mut [u64; N], source: [u64; N]) -> Result mod tests { use super::*; + #[test] + fn cached_storage_fits_the_unchanged_cold_state_envelope() -> Result<()> { + let mut decoder = PictureDecoder::new(DecodeLimits::default()); + let cold = decoder.fixed_memory()?; + for stage in 0..3 { + let capacity = decoder.session.capacity().max(1); + let reserved = (capacity + 1) * core::mem::size_of::>>(); + let physical = capacity * core::mem::size_of::(); + let active = decoder + .session + .first() + .map_or(0, |session| session.pictures.frame.active.capacity()); + assert!((capacity + 1) * core::mem::size_of::() <= reserved); + assert!(physical + (active + 1) * core::mem::size_of::() <= reserved); + assert_eq!(decoder.fixed_memory()?, cold); + if stage == 0 { + decoder.initialize()?; + } + if stage == 1 { + decoder + .session + .first_mut() + .ok_or(Error::Internal("test_session"))? + .pictures + .frame + .prepare()?; + } + } + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::>>>() + ); + Ok(()) + } + + #[test] + fn output_capacity_evicts_idle_memory_without_shifting_the_cold_limit() -> Result<()> { + struct Idle(Vec); + impl ReclaimMemory for Idle { + fn retained_bytes(&mut self, mandatory: usize, limit: usize) -> Result { + if mandatory + self.0.capacity() > limit { + self.0 = Vec::new(); + } + Ok(self.0.capacity()) + } + } + let mut idle = Idle(vec![0; 128]); + let mut reserved = 0; + { + let mut memory = OutputMemory { + codec_bytes: 128, + reserved: &mut reserved, + limit: 256, + reclaim: Some(&mut idle), + }; + memory.reserve(128)?; + assert_eq!( + memory.reserve(1), + Err(Error::Limit("reconstruction_memory")) + ); + } + assert_eq!(reserved, 128); + assert_eq!(idle.0.capacity(), 0); + Ok(()) + } + #[test] fn skipped_hash_messages_never_receive_checked_sample_credit() -> Result<()> { let mut decoder = PictureDecoder::new(DecodeLimits::default()); diff --git a/crates/coreav-hevc/src/lib.rs b/crates/coreav-hevc/src/lib.rs index 4e9ecaaf7..81b3f09ad 100644 --- a/crates/coreav-hevc/src/lib.rs +++ b/crates/coreav-hevc/src/lib.rs @@ -52,6 +52,8 @@ pub enum Error { Unsupported(&'static str), /// A caller budget or a fallible allocation was exhausted. Limit(&'static str), + /// A decoder invariant or caught reconstruction panic, not a stream failure. + Internal(&'static str), } impl core::fmt::Display for Error { @@ -60,6 +62,7 @@ impl core::fmt::Display for Error { Self::InvalidData(reason) => write!(f, "invalid data: {reason}"), Self::Unsupported(reason) => write!(f, "unsupported: {reason}"), Self::Limit(reason) => write!(f, "limit: {reason}"), + Self::Internal(reason) => write!(f, "internal decoder error: {reason}"), } } } diff --git a/crates/coreav-hevc/src/parser.rs b/crates/coreav-hevc/src/parser.rs index b0d590d76..f5abcd769 100644 --- a/crates/coreav-hevc/src/parser.rs +++ b/crates/coreav-hevc/src/parser.rs @@ -223,6 +223,9 @@ pub(crate) struct SliceInput<'a> { } pub(crate) trait PictureObserver { + fn reserve_scratch(&mut self, _scratch_bytes: usize, _dpb: &Dpb) -> Result<()> { + Ok(()) + } fn boundary(&mut self, dpb: &mut Dpb, scratch_bytes: usize) -> Result<()>; fn slice(&mut self, input: SliceInput<'_>, dpb: &Dpb) -> Result<()>; fn sei(&mut self, info: SeiInfo, dpb: &Dpb, scratch_bytes: usize) -> Result<()>; @@ -424,7 +427,13 @@ impl StreamCore { if bytes.len().saturating_sub(2) > memory_limit { return Err(Error::Limit("reconstruction_memory")); } + if let Some(observer) = observer.as_mut() { + observer.reserve_scratch(bytes.len().saturating_sub(2), &self.dpb)?; + } let rbsp = nal::rbsp(bytes, &self.limits)?; + if let Some(observer) = observer.as_mut() { + observer.reserve_scratch(rbsp.capacity(), &self.dpb)?; + } let mut bits = Bits::new(&rbsp); let info = match nal.unit_type { 32 => { @@ -515,8 +524,27 @@ impl StreamCore { } else { None }; - let parsed = - slice::parse(&mut bits, nal, prefix, s, p, &self.limits, retain_entries)?; + let mut reserve_entries = |bytes: usize| -> Result<()> { + if let Some(observer) = observer.as_mut() { + observer.reserve_scratch( + rbsp.capacity() + .checked_add(bytes) + .ok_or(Error::Limit("reconstruction_memory"))?, + &self.dpb, + )?; + } + Ok(()) + }; + let parsed = slice::parse( + &mut bits, + nal, + prefix, + s, + p, + &self.limits, + retain_entries, + &mut reserve_entries, + )?; let header = parsed.header; if header.rbsp_header_bytes >= rbsp.len() { return Err(Error::InvalidData("missing_slice_data")); diff --git a/crates/coreav-hevc/src/reconstruction.rs b/crates/coreav-hevc/src/reconstruction.rs index 2a78d4eb6..aa7f56740 100644 --- a/crates/coreav-hevc/src/reconstruction.rs +++ b/crates/coreav-hevc/src/reconstruction.rs @@ -5,8 +5,16 @@ use coreav_kernels_hevc::{self as kernels, BitDepth, IntraMode, MotionVector}; mod filter; mod inter; +mod parallel; mod syntax; +pub(crate) use parallel::Cache; + +pub(crate) fn caught(work: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)) + .unwrap_or(Err(Error::Internal("reconstruction_panic"))) +} + /// Effective scalar inter reconstruction work, without timing. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct InterStats { @@ -78,25 +86,57 @@ impl Motion { } } -pub(crate) struct Memory { +pub(crate) trait ReclaimMemory { + fn retained_bytes(&mut self, mandatory: usize, limit: usize) -> Result; +} + +pub(crate) struct Memory<'a> { pub(crate) used: usize, limit: usize, + peak: usize, + reclaim: Option<&'a mut dyn ReclaimMemory>, } -impl Memory { +impl<'a> Memory<'a> { pub(crate) fn new(used: usize, limit: usize) -> Result { - if used > limit { - return Err(Error::Limit("reconstruction_memory")); - } - Ok(Self { used, limit }) + Self::with_reclaim(used, limit, None) + } + + pub(crate) fn with_reclaim( + used: usize, + limit: usize, + reclaim: Option<&'a mut dyn ReclaimMemory>, + ) -> Result { + let mut memory = Self { + used: 0, + limit, + peak: 0, + reclaim, + }; + memory.charge(used)?; + Ok(memory) + } + + pub(crate) fn peak(&self) -> usize { + self.peak } pub(crate) fn charge(&mut self, bytes: usize) -> Result<()> { - self.used = self + let mandatory = self .used .checked_add(bytes) .filter(|n| *n <= self.limit) .ok_or(Error::Limit("reconstruction_memory"))?; + let retained = match &mut self.reclaim { + Some(reclaim) => reclaim.retained_bytes(mandatory, self.limit)?, + None => 0, + }; + let physical = mandatory + .checked_add(retained) + .filter(|n| *n <= self.limit) + .ok_or(Error::Limit("reconstruction_memory"))?; + self.used = mandatory; + self.peak = self.peak.max(physical); Ok(()) } @@ -177,6 +217,9 @@ fn allocation_bytes(values: &Vec) -> Result { } pub(crate) struct Frame { + #[cfg(test)] + test_hook: Option>, + pub(crate) workers: usize, pub(crate) deadline: Option, index: u64, info: PictureInfo, @@ -293,7 +336,7 @@ fn sample_bytes(planes: &[Vec; 3], cells: usize) -> Result { impl Frame { pub(crate) fn new(input: &SliceInput<'_>, fixed: usize, limit: usize) -> Result { Self::admit(input)?; - Self::allocate(input, None, fixed, limit) + Self::allocate(input, None, fixed, limit, None) } pub(crate) fn new_full( @@ -302,12 +345,14 @@ impl Frame { references: References, fixed: usize, limit: usize, + reclaim: Option<&mut dyn ReclaimMemory>, ) -> Result { Self::allocate( input, Some(ReferenceState { poc, references }), fixed, limit, + reclaim, ) } @@ -316,11 +361,12 @@ impl Frame { reference: Option, fixed: usize, limit: usize, + reclaim: Option<&mut dyn ReclaimMemory>, ) -> Result { if !input.header.first || input.header.address != 0 { return Err(Error::InvalidData("selected_picture_start")); } - let mut memory = Memory::new(fixed, limit)?; + let mut memory = Memory::with_reclaim(fixed, limit, reclaim)?; memory.charge(input.scratch_bytes)?; let width = input.picture.width as usize; let height = input.picture.height as usize; @@ -335,6 +381,9 @@ impl Frame { .ok_or(Error::Limit("picture_cells"))?; Ok(Self { deadline: None, + #[cfg(test)] + test_hook: None, + workers: 1, index: input.index, info: input.picture, depth: BitDepth::new(input.picture.bit_depth as u8)?, @@ -425,10 +474,20 @@ impl Frame { } pub(crate) fn hash(&mut self, sei: SeiInfo, fixed: usize, limit: usize) -> Result { + self.hash_reclaiming(sei, fixed, limit, None) + } + + pub(crate) fn hash_reclaiming( + &mut self, + sei: SeiInfo, + fixed: usize, + limit: usize, + reclaim: Option<&mut dyn ReclaimMemory>, + ) -> Result { if sei.picture_hash.is_none() && sei.reserved_hashes == 0 { return Ok(0); } - let peak = self.finalize(fixed, limit)?; + let peak = self.finalize_reclaiming(fixed, limit, reclaim)?; if !self.complete() || self.hash_seen { return Err(Error::InvalidData("selected_picture_hash_placement")); } diff --git a/crates/coreav-hevc/src/reconstruction/filter.rs b/crates/coreav-hevc/src/reconstruction/filter.rs index e8d8a38e3..bce7d7106 100644 --- a/crates/coreav-hevc/src/reconstruction/filter.rs +++ b/crates/coreav-hevc/src/reconstruction/filter.rs @@ -1,4 +1,4 @@ -use super::{Cell, Frame, Memory, Motion, MotionReference}; +use super::{Cell, Frame, Memory, Motion, MotionReference, ReclaimMemory}; use crate::{Error, Result, dpb::References}; use coreav_kernels_hevc::{self as kernels, BoundaryStrength, SaoBand, SaoEdge}; @@ -93,6 +93,15 @@ pub(super) struct Ctu { impl Frame { pub(crate) fn finalize(&mut self, fixed: usize, limit: usize) -> Result { + self.finalize_reclaiming(fixed, limit, None) + } + + pub(crate) fn finalize_reclaiming( + &mut self, + fixed: usize, + limit: usize, + mut reclaim: Option<&mut dyn ReclaimMemory>, + ) -> Result { if !self.complete() || self.filter_ctus.iter().any(|c| !c.seen) { return Err(Error::InvalidData("selected_picture_incomplete")); } @@ -118,7 +127,14 @@ impl Frame { .iter() .any(|ctu| ctu.sao.get(component).is_some_and(|s| s.kind > 1)) { - let mut memory = Memory::new(fixed, limit)?; + let mut memory = Memory::with_reclaim( + fixed, + limit, + match &mut reclaim { + Some(reclaim) => Some(&mut **reclaim), + None => None, + }, + )?; memory.charge(self.heap_bytes()?)?; let plane = self .planes @@ -126,7 +142,7 @@ impl Frame { .ok_or(Error::InvalidData("filter_component"))?; let mut source = memory.allocate(plane.len(), 0u16)?; source.copy_from_slice(plane); - peak = peak.max(memory.used); + peak = peak.max(memory.peak()); Some(source) } else { None @@ -543,6 +559,8 @@ mod tests { }; let retained = RetainedPicture::for_test(0, info, None); Ok(Frame { + test_hook: None, + workers: 1, deadline: None, index: 0, info, @@ -837,6 +855,8 @@ mod tests { }; let retained = RetainedPicture::for_test(0, info, None); let mut frame = Frame { + test_hook: None, + workers: 1, deadline: None, index: 0, info, diff --git a/crates/coreav-hevc/src/reconstruction/inter.rs b/crates/coreav-hevc/src/reconstruction/inter.rs index 4d7935fe7..535eafe89 100644 --- a/crates/coreav-hevc/src/reconstruction/inter.rs +++ b/crates/coreav-hevc/src/reconstruction/inter.rs @@ -1820,8 +1820,14 @@ mod tests { }; 3]; inter.weights = Some(weights); let input = fixture.input(1, inter); - let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + let mut frame = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + 1_000_000, + None, + )?; for cell in &mut frame.cells { cell.slice = 1; } @@ -1926,8 +1932,14 @@ mod tests { }; 3]; inter.weights = Some(weights); let input = fixture.input(0, inter); - let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + let mut frame = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + 1_000_000, + None, + )?; for cell in &mut frame.cells { cell.slice = 1; } @@ -1976,8 +1988,14 @@ mod tests { vector: vector(40, -8), }; for partition in [Partition::Horizontal, Partition::Vertical] { - let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + let mut frame = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + 1_000_000, + None, + )?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2014,7 +2032,7 @@ mod tests { ); } let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2047,7 +2065,7 @@ mod tests { let mut input = fixture.input(1, controls([1, 0])); input.pps = &pps; let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2079,7 +2097,7 @@ mod tests { let fixture = Fixture::new(8, 2)?; let input = fixture.input(1, controls([1, 0])); let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2127,8 +2145,14 @@ mod tests { pps.parallel_merge_log2 = level; let mut input = fixture.input(1, inter); input.pps = &pps; - let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + let mut frame = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + 1_000_000, + None, + )?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2178,7 +2202,7 @@ mod tests { })?; let input = fixture.input(1, inter); let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; let walker = syntax::Walker { frame: &mut frame, input: &input, @@ -2210,7 +2234,7 @@ mod tests { let dpb = fixture.dpb()?; let input = fixture.input(0, controls([1, 1])); let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; for cell in &mut frame.cells { cell.slice = 1; } @@ -2259,7 +2283,7 @@ mod tests { }); let input = fixture.input(0, inter); let mut frame = - super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000)?; + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; let walker = syntax::Walker { frame: &mut frame, input: &input, diff --git a/crates/coreav-hevc/src/reconstruction/parallel.rs b/crates/coreav-hevc/src/reconstruction/parallel.rs new file mode 100644 index 000000000..1a58d4c22 --- /dev/null +++ b/crates/coreav-hevc/src/reconstruction/parallel.rs @@ -0,0 +1,785 @@ +use super::{ + Cell, Frame, InterStats, Memory, ReclaimMemory, caught, + filter::Ctu, + syntax::{Checkpoint, Contexts, Qp}, +}; +use crate::{DecodeSummary, Error, IntraStats, Result, cabac::Cabac, dpb::Dpb, parser::SliceInput}; +use core::ops::Range; +use std::sync::{ + Condvar, Mutex, MutexGuard, + atomic::{AtomicUsize, Ordering}, +}; + +const STACK_BYTES: usize = 2 << 20; + +#[derive(Default)] +pub(crate) struct Cache { + frames: Vec, + peak: usize, + #[cfg(test)] + hits: usize, +} + +impl Cache { + #[cfg(test)] + pub(crate) fn hits(&self) -> usize { + self.hits + } + pub(crate) fn clear(&mut self) { + self.frames = Vec::new(); + } + + pub(crate) fn peak(&self) -> usize { + self.peak + } + + fn bytes(&self) -> Result { + self.frames + .iter() + .try_fold(super::allocation_bytes(&self.frames)?, |total, frame| { + total + .checked_add(frame.heap_bytes()?) + .ok_or(Error::Limit("reconstruction_memory")) + }) + } + + fn take(&mut self, template: &Frame, count: usize) -> Result>> { + let heap = template.heap_bytes()?; + let reusable = self.frames.len() == count - 1 + && self.frames.capacity() == count - 1 + && self.frames.iter().all(|frame| { + frame.info.width == template.info.width + && frame.info.height == template.info.height + && frame.info.bit_depth == template.info.bit_depth + && frame.info.ctb_log2 == template.info.ctb_log2 + && frame.reference.is_some() == template.reference.is_some() + && frame.storage_lengths() == template.storage_lengths() + && frame.heap_bytes().is_ok_and(|bytes| bytes == heap) + }); + if reusable { + #[cfg(test)] + { + self.hits += 1; + } + Ok(Some(core::mem::take(&mut self.frames))) + } else { + self.clear(); + Ok(None) + } + } +} + +impl ReclaimMemory for Cache { + fn retained_bytes(&mut self, mandatory: usize, limit: usize) -> Result { + let retained = self.bytes().unwrap_or(usize::MAX); + let retained = if mandatory + .checked_add(retained) + .is_some_and(|bytes| bytes <= limit) + { + retained + } else { + self.clear(); + 0 + }; + let physical = mandatory + .checked_add(retained) + .filter(|bytes| *bytes <= limit) + .ok_or(Error::Limit("reconstruction_memory"))?; + self.peak = self.peak.max(physical); + Ok(retained) + } +} + +#[cfg(test)] +#[derive(Clone, Copy)] +pub(super) enum TestFault { + None, + Error, + Panic, + Deadline, +} + +#[cfg(test)] +#[derive(Default)] +struct TestProgress { + upper_paused: bool, + lower_waiting: bool, + lower_published: bool, + released: bool, +} + +#[cfg(test)] +pub(super) struct TestHook { + fault: TestFault, + delay: bool, + state: Mutex, + ready: Condvar, +} + +#[cfg(test)] +impl TestHook { + pub(super) fn new(fault: TestFault, delay: bool) -> Self { + Self { + fault, + delay, + state: Mutex::new(TestProgress::default()), + ready: Condvar::new(), + } + } + + pub(super) fn compute(&self, row: u32, column: u32) -> Result<()> { + if (row, column) == (1, 1) { + match self.fault { + TestFault::None | TestFault::Deadline => {} + TestFault::Error => return Err(Error::InvalidData("injected_row_error")), + TestFault::Panic => std::panic::resume_unwind(Box::new(())), + } + } + Ok(()) + } + + fn waiting(&self, row: usize, column: usize) -> Option { + if self.delay && (row, column) == (1, 1) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .lower_waiting = true; + self.ready.notify_all(); + if matches!(self.fault, TestFault::Deadline) { + return Some(std::time::Instant::now() + std::time::Duration::from_millis(20)); + } + } + None + } + + fn publication(&self, row: usize, column: usize) { + if !self.delay { + return; + } + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if (row, column) == (0, 2) { + state.upper_paused = true; + self.ready.notify_all(); + while !state.released { + state = self + .ready + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } else if (row, column) == (1, 1) { + state.lower_published = true; + self.ready.notify_all(); + } + } + + pub(super) fn release_after_wait(&self, delay: std::time::Duration) -> bool { + let state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (state, _) = self + .ready + .wait_timeout_while(state, std::time::Duration::from_secs(2), |s| { + !s.upper_paused || !s.lower_waiting + }) + .unwrap_or_else(std::sync::PoisonError::into_inner); + let reached = state.upper_paused && state.lower_waiting; + let (mut state, _) = self + .ready + .wait_timeout_while(state, delay, |s| !s.lower_published) + .unwrap_or_else(std::sync::PoisonError::into_inner); + let premature = state.lower_published; + state.released = true; + self.ready.notify_all(); + reached && !premature + } +} + +#[derive(Clone, Copy)] +struct Continuation { + next_ctu: u32, + closed: bool, + contexts: Contexts, + checkpoint: Option, + qp: Qp, + stats: IntraStats, + inter: InterStats, +} + +struct Outcome { + result: Result, + consumed: u64, +} + +struct Band { + planes: [Vec; 3], + cells: Vec, + ctus: Vec, + y: usize, + height: usize, + completed: usize, + checkpoint: Option, + outcome: Option, +} + +struct Row { + band: Mutex, + ready: Condvar, +} + +impl Row { + fn lock(&self) -> MutexGuard<'_, Band> { + self.band + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +fn reserve(values: &mut Vec, count: usize, memory: &mut Memory) -> Result<()> { + memory.charge( + count + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; + values + .try_reserve_exact(count) + .map_err(|_| Error::Limit("picture_allocation"))?; + memory.charge( + (values.capacity() - count) + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + ) +} + +fn copy_rows( + source: &[T], + destination: &mut [T], + width: usize, + source_y: usize, + destination_y: usize, + height: usize, + columns: Range, +) -> Result<()> { + for y in 0..height { + let source = source + .get((source_y + y) * width + columns.start..(source_y + y) * width + columns.end) + .ok_or(Error::InvalidData("wpp_row_storage"))?; + destination + .get_mut( + (destination_y + y) * width + columns.start + ..(destination_y + y) * width + columns.end, + ) + .ok_or(Error::InvalidData("wpp_row_storage"))? + .copy_from_slice(source); + } + Ok(()) +} + +impl Band { + fn new(frame: &Frame, y: usize, ctb: usize, width: usize, memory: &mut Memory) -> Result { + let height = ctb.min(frame.info.height as usize - y); + let pixels = frame.info.width as usize; + Ok(Self { + planes: [ + memory.allocate(pixels * height, 0)?, + memory.allocate(pixels * height / 4, 0)?, + memory.allocate(pixels * height / 4, 0)?, + ], + cells: memory.allocate( + pixels * height / 16, + *frame + .cells + .first() + .ok_or(Error::InvalidData("wpp_row_storage"))?, + )?, + ctus: memory.allocate(width, Ctu::default())?, + y, + height, + completed: 0, + checkpoint: None, + outcome: None, + }) + } + + fn publish(&mut self, frame: &Frame, columns: Range) -> Result<()> { + let ctb = 1usize << frame.info.ctb_log2; + let width = frame.info.width as usize; + for (component, (source, destination)) in + frame.planes.iter().zip(&mut self.planes).enumerate() + { + let divisor = if component == 0 { 1 } else { 2 }; + copy_rows( + source, + destination, + width / divisor, + self.y / divisor, + 0, + self.height / divisor, + columns.start * ctb / divisor..(columns.end * ctb).min(width) / divisor, + )?; + } + copy_rows( + &frame.cells, + &mut self.cells, + width / 4, + self.y / 4, + 0, + self.height / 4, + columns.start * ctb / 4..(columns.end * ctb).min(width) / 4, + )?; + let row = self.y / ctb; + let ctus = width.div_ceil(ctb); + self.ctus + .get_mut(columns.clone()) + .ok_or(Error::InvalidData("wpp_row_storage"))? + .copy_from_slice( + frame + .filter_ctus + .get(row * ctus + columns.start..row * ctus + columns.end) + .ok_or(Error::InvalidData("wpp_row_storage"))?, + ); + Ok(()) + } + + fn import(&self, frame: &mut Frame, columns: Range, lines: Range) -> Result<()> { + let ctb = 1usize << frame.info.ctb_log2; + let width = frame.info.width as usize; + for (component, (source, destination)) in + self.planes.iter().zip(&mut frame.planes).enumerate() + { + let divisor = if component == 0 { 1 } else { 2 }; + copy_rows( + source, + destination, + width / divisor, + lines.start / divisor, + (self.y + lines.start) / divisor, + lines.len() / divisor, + columns.start * ctb / divisor..(columns.end * ctb).min(width) / divisor, + )?; + } + copy_rows( + &self.cells, + &mut frame.cells, + width / 4, + lines.start / 4, + (self.y + lines.start) / 4, + lines.len() / 4, + columns.start * ctb / 4..(columns.end * ctb).min(width) / 4, + )?; + let row = self.y / ctb; + let ctus = width.div_ceil(ctb); + frame + .filter_ctus + .get_mut(row * ctus + columns.start..row * ctus + columns.end) + .ok_or(Error::InvalidData("wpp_row_storage"))? + .copy_from_slice( + self.ctus + .get(columns) + .ok_or(Error::InvalidData("wpp_row_storage"))?, + ); + Ok(()) + } +} + +impl Frame { + fn storage_lengths(&self) -> [usize; 14] { + let [y, cb, cr] = self.planes.each_ref().map(Vec::len); + let [first, second] = self.scratch.predictors.each_ref().map(Vec::len); + [ + y, + cb, + cr, + self.cells.len(), + self.filter_ctus.len(), + self.scratch.coefficients.len(), + self.scratch.residual.len(), + self.scratch.prediction.len(), + self.scratch.output.len(), + self.scratch.scaling.len(), + self.scratch.references.len(), + first, + second, + self.scratch.inter_output.len(), + ] + } + + fn reset_private(&mut self, source: &Frame) { + let Self { + #[cfg(test)] + test_hook, + workers, + deadline, + index, + info, + depth, + planes, + cells, + filter_ctus, + filtered, + next_ctu, + segment_closed, + contexts, + wpp, + qp, + stats, + inter_stats, + reference, + hash_seen, + scratch, + } = self; + #[cfg(test)] + { + *test_hook = source.test_hook.clone(); + } + *workers = 1; + *deadline = source.deadline; + *index = source.index; + *info = source.info; + *depth = source.depth; + for (target, source) in planes.iter_mut().zip(&source.planes) { + target.copy_from_slice(source); + } + cells.copy_from_slice(&source.cells); + filter_ctus.copy_from_slice(&source.filter_ctus); + *filtered = source.filtered; + *next_ctu = source.next_ctu; + *segment_closed = source.segment_closed; + *contexts = source.contexts; + *wpp = source.wpp; + *qp = source.qp; + *stats = source.stats; + *inter_stats = source.inter_stats; + *reference = source.reference; + *hash_seen = source.hash_seen; + let super::Scratch { + coefficients, + residual, + prediction, + output, + scaling, + references, + predictors, + inter_output, + } = scratch; + coefficients.copy_from_slice(&source.scratch.coefficients); + residual.copy_from_slice(&source.scratch.residual); + prediction.copy_from_slice(&source.scratch.prediction); + output.copy_from_slice(&source.scratch.output); + scaling.copy_from_slice(&source.scratch.scaling); + references.copy_from_slice(&source.scratch.references); + for (target, source) in predictors.iter_mut().zip(&source.scratch.predictors) { + target.copy_from_slice(source); + } + inter_output.copy_from_slice(&source.scratch.inter_output); + } + + #[expect( + clippy::too_many_arguments, + reason = "same slice and resource boundary as serial reconstruction" + )] + pub(super) fn decode_wpp( + &mut self, + input: &SliceInput<'_>, + dpb: Option<&Dpb>, + remaining: &mut u64, + ranges: &[Range], + fixed: usize, + limit: usize, + threads: usize, + mut cache: Option<&mut Cache>, + ) -> Result> { + if threads <= 1 + || cfg!(target_family = "wasm") + || !input.pps.wpp + || input.pps.tiles + || !input.header.first + || input.header.dependent + || self.next_ctu != 0 + || ranges.len() != input.layout.height as usize + || ranges.len() < 2 + { + return Ok(None); + } + let reserved_bins = ranges.iter().try_fold(0u64, |total, range| { + total.checked_add(Cabac::max_bins(range.len())) + }); + if reserved_bins.is_none_or(|bins| bins > *remaining) { + return Ok(None); + } + let frame_bytes = self.heap_bytes()?; + let maximum = threads.clamp(1, 16).min(ranges.len()); + let count = (2..=maximum).rev().find(|count| { + frame_bytes + .checked_mul(*count) + .and_then(|n| { + n.checked_add((*count - 1) * (STACK_BYTES + core::mem::size_of::())) + }) + .and_then(|n| n.checked_add(ranges.len() * core::mem::size_of::())) + .and_then(|n| n.checked_add(fixed)) + .is_some_and(|n| n <= limit) + }); + let Some(count) = count else { return Ok(None) }; + let mut reusable = match &mut cache { + Some(cache) => cache.take(self, count)?, + None => None, + }; + let setup = (|| { + let mut memory = Memory::new(fixed, limit)?; + let warm = reusable.is_some(); + let mut workers = reusable.take().unwrap_or_default(); + if warm { + memory.charge(super::allocation_bytes(&workers)?)?; + for frame in &workers { + memory.charge(STACK_BYTES)?; + memory.charge(frame.heap_bytes()?)?; + } + } + let mut rows = Vec::new(); + reserve(&mut rows, ranges.len(), &mut memory)?; + for y in 0..ranges.len() { + rows.push(Row { + band: Mutex::new(Band::new( + self, + y << input.sps.ctb_log2, + 1 << input.sps.ctb_log2, + input.layout.width as usize, + &mut memory, + )?), + ready: Condvar::new(), + }); + } + if warm { + let mut cold = Memory::new(memory.used, limit)?; + cold.charge(input.scratch_bytes)?; + for frame in &mut workers { + frame.reset_private(self); + } + } else { + reserve(&mut workers, count - 1, &mut memory)?; + for _ in 1..count { + memory.charge(STACK_BYTES)?; + let mut frame = + Self::allocate(input, self.reference, memory.used, limit, None)?; + frame.deadline = self.deadline; + #[cfg(test)] + { + frame.test_hook = self.test_hook.clone(); + } + memory.charge(frame.heap_bytes()?)?; + workers.push(frame); + } + } + Ok((rows, workers, memory.used)) + })(); + let (rows, mut workers, peak) = match setup { + Ok(setup) => setup, + Err(Error::Limit(_)) => return Ok(None), + Err(error) => return Err(error), + }; + let next = AtomicUsize::new(0); + self.workers = std::thread::scope(|scope| { + let mut started = 1; + for frame in &mut workers { + let (rows, next) = (&rows, &next); + if std::thread::Builder::new() + .stack_size(STACK_BYTES) + .spawn_scoped(scope, move || { + work(frame, input, dpb, ranges, rows, next); + }) + .is_ok() + { + started += 1; + } + } + work(self, input, dpb, ranges, &rows, &next); + started + }); + let mut summary = DecodeSummary::default(); + for row in rows { + let mut band = row + .band + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let outcome = band + .outcome + .take() + .ok_or(Error::InvalidData("wpp_unfinished_row"))?; + if outcome.consumed > *remaining { + *remaining = 0; + return Err(Error::Limit("cabac_bin_budget")); + } + *remaining -= outcome.consumed; + let continuation = outcome.result?; + band.import(self, 0..band.completed, 0..band.height)?; + summary.add(continuation.stats, continuation.inter)?; + self.next_ctu = continuation.next_ctu; + self.segment_closed = continuation.closed; + self.contexts = continuation.contexts; + self.wpp = continuation.checkpoint; + self.qp = continuation.qp; + } + self.stats = summary.reconstruction; + self.inter_stats = summary.inter; + if let Some(cache) = cache { + cache.frames = workers; + } + Ok(Some(peak)) + } +} + +fn work( + frame: &mut Frame, + input: &SliceInput<'_>, + dpb: Option<&Dpb>, + ranges: &[Range], + rows: &[Row], + next: &AtomicUsize, +) { + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(row) = rows.get(index) else { return }; + let allowance = ranges + .get(index) + .map_or(0, |range| Cabac::max_bins(range.len())); + frame.next_ctu = index as u32 * input.layout.width; + frame.segment_closed = false; + frame.stats = IntraStats::default(); + frame.inter_stats = InterStats::default(); + frame.wpp = None; + let mut budget = allowance; + let result = caught(|| decode_row(frame, input, dpb, ranges, rows, index, &mut budget)) + .map(|()| Continuation { + next_ctu: frame.next_ctu, + closed: frame.segment_closed, + contexts: frame.contexts, + checkpoint: frame.wpp, + qp: frame.qp, + stats: frame.stats, + inter: frame.inter_stats, + }); + row.lock().outcome = Some(Outcome { + result, + consumed: allowance - budget, + }); + row.ready.notify_all(); + } +} + +fn decode_row( + frame: &mut Frame, + input: &SliceInput<'_>, + dpb: Option<&Dpb>, + ranges: &[Range], + rows: &[Row], + index: usize, + budget: &mut u64, +) -> Result<()> { + let row = rows + .get(index) + .ok_or(Error::InvalidData("wpp_row_storage"))?; + let range = ranges + .get(index) + .ok_or(Error::InvalidData("intra_substream_range"))?; + let bytes = input + .rbsp + .get(range.clone()) + .ok_or(Error::InvalidData("intra_substream_range"))?; + let width = input.layout.width as usize; + let slice = input + .syntax + .independent_address + .checked_add(1) + .ok_or(Error::InvalidData("intra_slice_address"))?; + let mut cabac = Cabac::new(bytes, budget)?; + for x in 0..width { + if let Some(above) = index.checked_sub(1).and_then(|index| rows.get(index)) { + #[cfg(test)] + if let Some(hook) = &frame.test_hook { + frame.deadline = hook.waiting(index, x).or(frame.deadline); + } + let mut band = above.lock(); + loop { + frame.check_deadline()?; + if let Some(outcome) = &band.outcome { + outcome.result?; + } + if band.completed >= (x + 2).min(width) { + break; + } + band = above + .ready + .wait_timeout(band, std::time::Duration::from_millis(10)) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + let columns = x.saturating_sub(1)..(x + 2).min(width); + band.import( + frame, + columns.clone(), + band.height.saturating_sub(4)..band.height, + )?; + if x == 0 { + band.import(frame, columns, 0..4.min(band.height))?; + frame.wpp = band.checkpoint; + } + } + frame.decode_ctu(input, dpb, slice, &mut cabac)?; + #[cfg(test)] + if let Some(hook) = &frame.test_hook { + hook.publication(index, x); + } + { + let mut band = row.lock(); + band.publish(frame, x..x + 1)?; + band.completed = x + 1; + band.checkpoint = frame.wpp; + } + row.ready.notify_all(); + if cabac.terminate()? { + if index + 1 != ranges.len() { + return Err(Error::InvalidData("intra_unconsumed_substreams")); + } + super::syntax::end_range(&mut cabac, bytes.len(), true)?; + frame.segment_closed = true; + return Ok(()); + } + if frame.next_ctu == input.layout.width * input.layout.height { + return Err(Error::InvalidData("intra_picture_termination")); + } + if x + 1 == width { + if !cabac.terminate()? { + return Err(Error::InvalidData("end_of_subset_one_bit")); + } + super::syntax::end_range(&mut cabac, bytes.len(), false)?; + } + } + if index + 1 == ranges.len() { + Err(Error::InvalidData("intra_missing_substream")) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod cache_tests { + use super::*; + + #[test] + fn empty_allocated_worker_slots_are_charged_and_revocable() -> Result<()> { + let mut cache = Cache::default(); + cache + .frames + .try_reserve_exact(3) + .map_err(|_| Error::Limit("picture_allocation"))?; + let bytes = cache.frames.capacity() * core::mem::size_of::(); + assert!(bytes > 0); + assert_eq!(cache.retained_bytes(0, bytes)?, bytes); + assert_eq!(cache.retained_bytes(0, bytes - 1)?, 0); + assert_eq!(cache.frames.capacity(), 0); + Ok(()) + } +} diff --git a/crates/coreav-hevc/src/reconstruction/syntax.rs b/crates/coreav-hevc/src/reconstruction/syntax.rs index 72ca7db99..fc7ca5bae 100644 --- a/crates/coreav-hevc/src/reconstruction/syntax.rs +++ b/crates/coreav-hevc/src/reconstruction/syntax.rs @@ -113,6 +113,7 @@ pub(super) struct Checkpoint { slice: u32, } +#[derive(Clone, Copy)] pub(super) struct Qp { current: i32, predictor: i32, @@ -375,7 +376,7 @@ fn alignment(cabac: &mut Cabac<'_, '_>) -> Result<()> { Ok(()) } -fn end_range(cabac: &mut Cabac<'_, '_>, bytes: usize, final_slice: bool) -> Result<()> { +pub(super) fn end_range(cabac: &mut Cabac<'_, '_>, bytes: usize, final_slice: bool) -> Result<()> { alignment(cabac)?; let remaining = bytes .checked_sub(cabac.position() / 8) @@ -561,9 +562,13 @@ impl Frame { limit: usize, ) -> Result { Self::admit(&input)?; - self.decode_common(input, None, remaining, fixed, limit) + self.decode_common(input, None, remaining, fixed, limit, 1, None) } + #[expect( + clippy::too_many_arguments, + reason = "shared slice and memory boundary" + )] pub(crate) fn decode_full( &mut self, input: SliceInput<'_>, @@ -571,6 +576,8 @@ impl Frame { remaining: &mut u64, fixed: usize, limit: usize, + threads: usize, + cache: Option<&mut super::Cache>, ) -> Result { if self.reference.is_none() || input.syntax.kind > 2 @@ -578,9 +585,13 @@ impl Frame { { return Err(Error::InvalidData("inter_segment_state")); } - self.decode_common(input, Some(dpb), remaining, fixed, limit) + self.decode_common(input, Some(dpb), remaining, fixed, limit, threads, cache) } + #[expect( + clippy::too_many_arguments, + reason = "shared slice and memory boundary" + )] fn decode_common( &mut self, input: SliceInput<'_>, @@ -588,6 +599,8 @@ impl Frame { remaining: &mut u64, fixed: usize, limit: usize, + threads: usize, + mut cache: Option<&mut super::Cache>, ) -> Result { if self.index != input.index || self.info != input.picture @@ -610,25 +623,40 @@ impl Frame { if count > (total_ctus - self.next_ctu) as usize { return Err(Error::InvalidData("intra_substream_count")); } - let mut memory = Memory::new(fixed, limit)?; - memory.charge(self.heap_bytes()?)?; - memory.charge(input.scratch_bytes)?; - memory.charge( - count - .checked_mul(core::mem::size_of::>()) - .ok_or(Error::Limit("reconstruction_memory"))?, - )?; - let ranges = substream::ranges( - input.original, - input.header.rbsp_header_bytes, - input.entry_lengths, - count, - )?; - memory.charge( - (ranges.capacity() - count) - .checked_mul(core::mem::size_of::>()) - .ok_or(Error::Limit("reconstruction_memory"))?, - )?; + let (ranges, fixed, prior_peak) = { + let mut memory = Memory::with_reclaim( + fixed, + limit, + match &mut cache { + Some(cache) => Some(&mut **cache), + None => None, + }, + )?; + memory.charge(self.heap_bytes()?)?; + memory.charge(input.scratch_bytes)?; + memory.charge( + count + .checked_mul(core::mem::size_of::>()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; + let ranges = substream::ranges( + input.original, + input.header.rbsp_header_bytes, + input.entry_lengths, + count, + )?; + memory.charge( + (ranges.capacity() - count) + .checked_mul(core::mem::size_of::>()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; + (ranges, memory.used, memory.peak()) + }; + if let Some(peak) = self.decode_wpp( + &input, dpb, remaining, &ranges, fixed, limit, threads, cache, + )? { + return Ok(peak.max(prior_peak)); + } if !input.header.dependent { self.contexts = Contexts::new(input.syntax.qp_y, input.syntax.initialization); self.qp = Qp::new(input.syntax.qp_y); @@ -649,36 +677,14 @@ impl Frame { .ok_or(Error::InvalidData("intra_substream_range"))?; let mut cabac = Cabac::new(bytes, remaining)?; loop { - self.check_deadline()?; - let location = input.layout.at_tile_scan(self.next_ctu)?; - self.start_ctu(&input, location, slice)?; - self.read_sao(&input, location, slice, &mut cabac)?; - let mut walker = Walker { - frame: self, - input: &input, - location, - slice, - dpb, - }; - walker.coding_tree( - &mut cabac, - Area { - x: (location.x << input.sps.ctb_log2) as usize, - y: (location.y << input.sps.ctb_log2) as usize, - log2: input.sps.ctb_log2, - }, - 0, - )?; - self.stats.ctus += 1; - self.save_wpp(&input, location, slice); - self.next_ctu += 1; + let location = self.decode_ctu(&input, dpb, slice, &mut cabac)?; if cabac.terminate()? { if range_index + 1 != ranges.len() { return Err(Error::InvalidData("intra_unconsumed_substreams")); } end_range(&mut cabac, bytes.len(), true)?; self.segment_closed = true; - return Ok(memory.used); + return Ok(prior_peak); } if self.next_ctu == total_ctus { return Err(Error::InvalidData("intra_picture_termination")); @@ -703,6 +709,43 @@ impl Frame { Err(Error::InvalidData("intra_missing_substream")) } + pub(super) fn decode_ctu( + &mut self, + input: &SliceInput<'_>, + dpb: Option<&crate::dpb::Dpb>, + slice: u32, + cabac: &mut Cabac<'_, '_>, + ) -> Result { + self.check_deadline()?; + let location = input.layout.at_tile_scan(self.next_ctu)?; + #[cfg(test)] + if let Some(hook) = &self.test_hook { + hook.compute(location.y, location.x)?; + } + self.start_ctu(input, location, slice)?; + self.read_sao(input, location, slice, cabac)?; + Walker { + frame: self, + input, + location, + slice, + dpb, + } + .coding_tree( + cabac, + Area { + x: (location.x << input.sps.ctb_log2) as usize, + y: (location.y << input.sps.ctb_log2) as usize, + log2: input.sps.ctb_log2, + }, + 0, + )?; + self.stats.ctus += 1; + self.save_wpp(input, location, slice); + self.next_ctu += 1; + Ok(location) + } + pub(super) fn cell_index(&self, x: usize, y: usize) -> Result { if x >= self.info.width as usize || y >= self.info.height as usize { return Err(Error::InvalidData("intra_cell_address")); @@ -2337,9 +2380,9 @@ mod tests { let input = inter_input(&fixture, &rbsp, &original, initialization); let dpb = reference_dpb(&input)?; let mut frame = - Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX)?; + Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX, None)?; let mut budget = bins; - let decoded = frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX); + let decoded = frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX, 1, None); assert!( decoded.is_ok(), "{initialization:?} skip={skipped} depth={depth}: {decoded:?}" @@ -2384,9 +2427,9 @@ mod tests { let input = inter_input(&fixture, &rbsp, &original, initialization); let dpb = reference_dpb(&input)?; let mut frame = - Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX)?; + Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX, None)?; let mut budget = 10; - frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX)?; + frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX, 1, None)?; assert_eq!(budget, 0); assert!(frame.complete()); assert!( @@ -2418,9 +2461,10 @@ mod tests { let original = escaped(&rbsp); let input = inter_input(&fixture, &rbsp, &original, initialization); let dpb = reference_dpb(&input)?; - let mut frame = Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX)?; + let mut frame = + Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX, None)?; let mut budget = 5; - frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX)?; + frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX, 1, None)?; assert_eq!(budget, 0); assert!(frame.complete()); assert!( @@ -2579,9 +2623,10 @@ mod tests { inter.mvd_l1_zero = zero; } let dpb = reference_dpb(&input)?; - let mut frame = Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX)?; + let mut frame = + Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX, None)?; let mut budget = 11; - let result = frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX); + let result = frame.decode_full(input, &dpb, &mut budget, 0, usize::MAX, 1, None); if !zero { assert!(result.is_err()); continue; @@ -3335,6 +3380,573 @@ mod tests { Ok(()) } + fn above_right_wavefront() -> Result<(Fixture, Vec, [u64; 1])> { + let fixture = fixture(48, 32, 8, true, 1)?; + let top = pcm_body( + &[[0x86, 0x80], [0x8b, 0x80], [0x91, 0x80]], + [0xfd, 0x80], + &[34, 51, 68], + )?; + let lengths = [top.len() as u64]; + let mut rbsp = vec![0x80]; + rbsp.extend(top); + rbsp.extend([0x90, 0x44, 0x2b, 0xb1, 0x80]); + Ok((fixture, rbsp, lengths)) + } + + #[test] + fn wavefront_above_right_prediction_waits_for_publication() -> Result<()> { + use super::super::parallel::{TestFault, TestHook}; + let (fixture, rbsp, lengths) = above_right_wavefront()?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 10_000_000; + serial.decode_common(input, None, &mut budget, 0, usize::MAX, 1, None)?; + assert_eq!(serial.planes[0][16 * 48 + 30], 55); + assert_eq!(serial.planes[0][16 * 48 + 31], 64); + assert_eq!(serial.stats.pcm_units, 3); + assert_eq!(serial.stats.coding_units, 6); + assert_eq!(serial.cell(16, 16)?.mode, Some(IntraMode::new(34)?)); + for threads in [2, 4, 8] { + let hook = std::sync::Arc::new(TestHook::new(TestFault::None, true)); + let mut frame = Frame::new(&input, 0, usize::MAX)?; + frame.test_hook = Some(hook.clone()); + let mut parallel_budget = 10_000_000; + std::thread::scope(|scope| { + let controller = + scope.spawn(|| hook.release_after_wait(std::time::Duration::from_millis(20))); + let result = frame.decode_common( + input, + None, + &mut parallel_budget, + 0, + usize::MAX, + threads, + None, + ); + assert!(controller.join().unwrap()); + result + })?; + assert!(frame.workers > 1); + assert!(frame.complete()); + assert_eq!(frame.planes, serial.planes); + assert_eq!(frame.stats, serial.stats); + assert_eq!(frame.inter_stats, serial.inter_stats); + assert_eq!(parallel_budget, budget); + } + Ok(()) + } + + #[test] + fn wavefront_mid_row_errors_and_panics_join_every_worker() -> Result<()> { + use super::super::parallel::{TestFault, TestHook}; + let (fixture, rbsp, lengths) = cached_wavefront(68, 8, 8)?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + let mut control = Frame::new(&input, 0, usize::MAX)?; + let control_result = control.decode(input, &mut 10_000_000, 0, usize::MAX); + assert!( + control_result.is_ok(), + "{control_result:?} at CTU {}", + control.next_ctu + ); + assert!(control.complete()); + assert_eq!(control.stats.ctus, 24); + for (fault, expected) in [ + (TestFault::Error, Error::InvalidData("injected_row_error")), + (TestFault::Panic, Error::Internal("reconstruction_panic")), + ] { + let mut serial_budget = None; + for threads in [1, 2, 4, 8] { + let mut frame = Frame::new(&input, 0, usize::MAX)?; + frame.test_hook = Some(std::sync::Arc::new(TestHook::new(fault, false))); + let mut budget = 10_000_000; + assert_eq!( + super::super::caught(|| frame.decode_common( + input, + None, + &mut budget, + 0, + usize::MAX, + threads, + None + )), + Err(expected) + ); + assert_eq!(frame.workers > 1, threads > 1); + assert_eq!(frame.workers, threads); + assert!(!frame.complete()); + if let Some(expected_budget) = serial_budget { + assert_eq!(budget, expected_budget); + } else { + serial_budget = Some(budget); + } + } + } + Ok(()) + } + + fn cached_wavefront(value: u8, depth: u32, rows: u32) -> Result<(Fixture, Vec, Vec)> { + let (_, mut rbsp, prefix_lengths) = above_right_wavefront()?; + let mut fixture = fixture(48, rows * 16, depth, true, 1)?; + fixture.sps.pcm = Some(Pcm { + luma_depth: 8, + chroma_depth: 8, + min_log2: 4, + max_log2: 4, + loop_filter_disabled: false, + }); + if rows == 2 { + return Ok((fixture, rbsp, prefix_lengths.to_vec())); + } + let mut lengths = vec![prefix_lengths[0], 5]; + let last = rbsp.len() - 2; + rbsp[last] = 0xb0; + for (row, starts) in [ + [0x9c, 0xa0, 0xa5], + [0xa6, 0xaa, 0xae], + [0xaf, 0xb2, 0xb6], + [0xb7, 0xb9, 0xbd], + [0xbe, 0xc0, 0xc3], + [0xc4, 0xc6, 0xc9], + ] + .into_iter() + .enumerate() + { + let body = pcm_body( + &starts.map(|byte| [byte, 0x80]), + [if row == 5 { 0xfe } else { 0xfd }, 0x80], + &[value; 3], + )?; + if row != 5 { + lengths.push(body.len() as u64); + } + rbsp.extend(body); + } + Ok((fixture, rbsp, lengths)) + } + + #[test] + fn wavefront_warm_workers_match_cold_and_serial_across_picture_changes() -> Result<()> { + use super::super::{Cache, ReclaimMemory}; + for threads in [2, 4, 8] { + let mut cache = Cache::default(); + for (index, (depth, rows)) in [ + (8, 8), + (8, 8), + (8, 8), + (8, 8), + (10, 8), + (10, 8), + (8, 2), + (8, 2), + (8, 8), + ] + .into_iter() + .enumerate() + { + let (fixture, rbsp, lengths) = cached_wavefront(41 + index as u8 * 7, depth, rows)?; + let original = escaped(&rbsp); + let mut input = fixture.input(segment(&rbsp, &original, &lengths)); + input.index = index as u64; + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut expected_budget = 10_000_000; + serial.decode(input, &mut expected_budget, 0, usize::MAX)?; + for warm in [false, true] { + let mut cold = Cache::default(); + let pool = if warm { &mut cache } else { &mut cold }; + let mut frame = Frame::allocate(&input, None, 0, usize::MAX, Some(pool))?; + let mut budget = 10_000_000; + frame.decode_common( + input, + None, + &mut budget, + 0, + usize::MAX, + threads, + Some(pool), + )?; + assert_eq!(frame.workers, threads.min(rows as usize)); + assert!(frame.complete()); + assert_eq!(frame.planes, serial.planes); + assert_eq!(frame.stats, serial.stats); + assert_eq!(frame.inter_stats, serial.inter_stats); + assert_eq!(budget, expected_budget); + assert!(pool.retained_bytes(0, usize::MAX)? > 0); + } + } + assert!(cache.hits() >= 4); + cache.clear(); + assert_eq!(cache.retained_bytes(0, usize::MAX)?, 0); + } + Ok(()) + } + + #[test] + fn wavefront_warm_errors_panics_and_deadlines_release_every_private_frame() -> Result<()> { + use super::super::{ + Cache, ReclaimMemory, + parallel::{TestFault, TestHook}, + }; + let (fixture, rbsp, lengths) = cached_wavefront(68, 8, 8)?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + for threads in [2, 4, 8] { + for (fault, expected) in [ + (TestFault::Error, Error::InvalidData("injected_row_error")), + (TestFault::Panic, Error::Internal("reconstruction_panic")), + (TestFault::Deadline, Error::Limit("decode_deadline")), + ] { + let mut cache = Cache::default(); + let mut first = Frame::new(&input, 0, usize::MAX)?; + first.decode_common( + input, + None, + &mut 10_000_000, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + let mut next = Frame::new(&input, 0, usize::MAX)?; + if matches!(fault, TestFault::Deadline) { + next.deadline = Some(std::time::Instant::now()); + } else { + next.test_hook = Some(std::sync::Arc::new(TestHook::new(fault, false))); + } + assert_eq!( + super::super::caught(|| next.decode_common( + input, + None, + &mut 10_000_000, + 0, + usize::MAX, + threads, + Some(&mut cache) + )), + Err(expected) + ); + assert_eq!(next.workers, threads); + assert!(!next.complete()); + assert_eq!(cache.hits(), 1); + assert_eq!(cache.retained_bytes(0, usize::MAX)?, 0); + } + } + Ok(()) + } + + #[test] + fn wavefront_warm_memory_preserves_independent_cold_admission_boundaries() -> Result<()> { + use super::super::{Cache, ReclaimMemory}; + fn cold_workers(input: SliceInput<'_>, threads: usize, limit: usize) -> Result { + let result = (|| { + let mut frame = Frame::new(&input, 0, limit)?; + frame.decode_common(input, None, &mut 10_000_000, 0, limit, threads, None)?; + Ok(frame.workers) + })(); + match result { + Err(Error::Limit("reconstruction_memory")) => Ok(0), + other => other, + } + } + let (fixture, rbsp, lengths) = cached_wavefront(68, 8, 8)?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + for extra in [0, 1 << 20] { + let mut input = input; + input.scratch_bytes += extra; + for threads in [2, 4, 8] { + let mut low = 0; + let mut high = 32 << 20; + assert_eq!(cold_workers(input, threads, high)?, threads); + while low < high { + let middle = low + (high - low) / 2; + if cold_workers(input, threads, middle)? == threads { + high = middle; + } else { + low = middle + 1; + } + } + for limit in [low - 1, low] { + let expected = cold_workers(input, threads, limit)?; + let mut cache = Cache::default(); + let mut seed = Frame::new(&input, 0, usize::MAX)?; + seed.decode_common( + input, + None, + &mut 10_000_000, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + drop(seed); + let mut frame = Frame::allocate(&input, None, 0, limit, Some(&mut cache))?; + frame.decode_common( + input, + None, + &mut 10_000_000, + 0, + limit, + threads, + Some(&mut cache), + )?; + assert_eq!(frame.workers, expected); + assert!(frame.complete()); + } + let frame = Frame::new(&input, 0, usize::MAX)?; + let constructor_limit = frame.heap_bytes()? + input.scratch_bytes; + drop(frame); + for limit in [constructor_limit - 1, constructor_limit] { + let mut cache = Cache::default(); + let mut seed = Frame::new(&input, 0, usize::MAX)?; + seed.decode_common( + input, + None, + &mut 10_000_000, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + drop(seed); + let cold = Frame::new(&input, 0, limit); + let warm = Frame::allocate(&input, None, 0, limit, Some(&mut cache)); + assert_eq!(warm.map(|_| ()), cold.map(|_| ())); + assert_eq!(cache.retained_bytes(0, usize::MAX)?, 0); + } + } + } + Ok(()) + } + + #[test] + fn wavefront_deadline_expires_inside_a_parallel_dependency_wait() -> Result<()> { + use super::super::parallel::{TestFault, TestHook}; + let (fixture, rbsp, lengths) = above_right_wavefront()?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + for threads in [2, 4, 8] { + let hook = std::sync::Arc::new(TestHook::new(TestFault::Deadline, true)); + let mut frame = Frame::new(&input, 0, usize::MAX)?; + frame.test_hook = Some(hook.clone()); + std::thread::scope(|scope| { + let controller = + scope.spawn(|| hook.release_after_wait(std::time::Duration::from_millis(150))); + assert_eq!( + frame.decode_common(input, None, &mut 10_000_000, 0, usize::MAX, threads, None), + Err(Error::Limit("decode_deadline")) + ); + assert!(controller.join().unwrap()); + }); + assert!(frame.workers > 1); + assert!(!frame.complete()); + } + Ok(()) + } + + #[test] + fn wavefront_workers_preserve_samples_work_and_failure_order() -> Result<()> { + for width in [1, 3] { + let fixture = fixture(width * 16, 32, 8, true, 1)?; + let (top, bottom) = if width == 1 { + ( + pcm_body(&[[0x86, 0x80]], [0xfd, 0x80], &[34])?, + pcm_body(&[[0x86, 0x80]], [0xfe, 0x80], &[51])?, + ) + } else { + ( + pcm_body( + &[[0x86, 0x80], [0x8b, 0x80], [0x91, 0x80]], + [0xfd, 0x80], + &[34; 3], + )?, + pcm_body( + &[[0x92, 0x80], [0x96, 0x80], [0x9b, 0x80]], + [0xfe, 0x80], + &[51; 3], + )?, + ) + }; + let lengths = [top.len() as u64]; + for truncate in [false, true] { + let mut rbsp = vec![0x80]; + rbsp.extend(&top); + rbsp.extend(&bottom); + if truncate { + rbsp.truncate(rbsp.len() - 7); + } + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + for allowance in (0..=24).chain([10_000_000]) { + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut budget = allowance; + let expected = + serial.decode_common(input, None, &mut budget, 0, usize::MAX, 1, None); + for threads in [2, 4, 8] { + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut parallel_budget = allowance; + let actual = frame.decode_common( + input, + None, + &mut parallel_budget, + 0, + usize::MAX, + threads, + None, + ); + assert_eq!(actual.map(|_| ()), expected.map(|_| ())); + assert_eq!(parallel_budget, budget); + if allowance < 10_000_000 { + assert_eq!(frame.workers, 1); + } + if actual.is_ok() { + assert_eq!(frame.workers > 1, allowance == 10_000_000); + assert_eq!(frame.planes, serial.planes); + assert_eq!(frame.stats, serial.stats); + assert_eq!(frame.inter_stats, serial.inter_stats); + assert_eq!(frame.complete(), serial.complete()); + assert_eq!(frame.next_ctu, serial.next_ctu); + } + } + } + } + } + Ok(()) + } + + #[test] + fn wavefront_memory_fallback_and_deadline_preserve_serial_admission() -> Result<()> { + let fixture = fixture(16, 32, 8, true, 1)?; + let top = pcm_body(&[[0x86, 0x80]], [0xfd, 0x80], &[34])?; + let lengths = [top.len() as u64]; + let mut rbsp = vec![0x80]; + rbsp.extend(top); + rbsp.extend(pcm_body(&[[0x86, 0x80]], [0xfe, 0x80], &[51])?); + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + for threads in [1, 2, 4, 8] { + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 100; + frame.decode_common(input, None, &mut budget, 0, 1 << 20, threads, None)?; + assert!(frame.complete()); + assert_eq!(frame.workers, 1); + let mut frame = Frame::new(&input, 0, usize::MAX)?; + frame.deadline = Some(std::time::Instant::now()); + assert_eq!( + frame.decode_common(input, None, &mut budget, 0, usize::MAX, threads, None), + Err(Error::Limit("decode_deadline")) + ); + } + Ok(()) + } + + #[test] + fn wavefront_eight_rows_finish_or_fail_without_stranding_workers() -> Result<()> { + let fixture = fixture(16, 128, 8, true, 1)?; + let mut rbsp = vec![0x80]; + let mut lengths = Vec::new(); + for row in 0..8 { + let body = pcm_body( + &[[0x86, 0x80]], + [if row == 7 { 0xfe } else { 0xfd }, 0x80], + &[34 + row], + )?; + if row < 7 { + lengths.push(body.len() as u64); + } + rbsp.extend(body); + } + for mutation in 0..3 { + let mut data = rbsp.clone(); + if mutation == 1 { + data.truncate(data.len() - 7); + } + if mutation == 2 { + data[1 + 2 * lengths[0] as usize] = 0xff; + } + let original = escaped(&data); + let input = fixture.input(segment(&data, &original, &lengths)); + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 10_000_000; + let expected = serial.decode_common(input, None, &mut budget, 0, usize::MAX, 1, None); + for threads in [2, 4, 8] { + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut parallel_budget = 10_000_000; + let actual = frame.decode_common( + input, + None, + &mut parallel_budget, + 0, + usize::MAX, + threads, + None, + ); + assert_eq!(actual.map(|_| ()), expected.map(|_| ())); + assert_eq!(parallel_budget, budget); + if actual.is_ok() { + assert_eq!(frame.workers, threads); + assert_eq!(frame.planes, serial.planes); + assert_eq!(frame.stats, serial.stats); + assert!(frame.complete()); + } + } + } + Ok(()) + } + + #[test] + fn wavefront_partial_final_row_continues_as_a_dependent_segment() -> Result<()> { + let fixture = fixture(48, 32, 8, true, 1)?; + let top = pcm_body( + &[[0x86, 0x80], [0x8b, 0x80], [0x91, 0x80]], + [0xfd, 0x80], + &[34; 3], + )?; + let bottom = pcm_body(&[[0x92, 0x80]], [0xfe, 0x80], &[51])?; + let lengths = [top.len() as u64]; + let mut rbsp = vec![0x80]; + rbsp.extend(top); + rbsp.extend(bottom); + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + let mut expected = None; + for threads in [1, 2, 4, 8] { + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 10_000_000; + frame.decode_common(input, None, &mut budget, 0, usize::MAX, threads, None)?; + assert_eq!(frame.next_ctu, 4); + let mut tail = vec![0x80]; + tail.extend(pcm_body( + &[[0x97, 0x80], [0x9b, 0x80]], + [0xfe, 0x80], + &[68; 2], + )?); + let nal = escaped(&tail); + let mut segment = segment(&tail, &nal, &[]); + segment.address = 4; + segment.dependent = true; + frame.decode_common( + fixture.input(segment), + None, + &mut budget, + 0, + usize::MAX, + threads, + None, + )?; + assert!(frame.complete()); + let actual = (frame.planes, frame.stats, budget); + if let Some(expected) = &expected { + assert_eq!(&actual, expected); + } else { + expected = Some(actual); + } + } + Ok(()) + } + #[test] fn wpp_slices_starting_inside_a_row_only_restore_an_available_above_right() -> Result<()> { for start in [1, 2] { @@ -3501,6 +4113,56 @@ mod tests { Ok(()) } + #[test] + fn sao_evicts_idle_storage_at_the_source_allocation() -> Result<()> { + struct Idle(Vec); + impl crate::reconstruction::ReclaimMemory for Idle { + fn retained_bytes(&mut self, mandatory: usize, limit: usize) -> Result { + if mandatory + .checked_add(self.0.capacity()) + .is_none_or(|bytes| bytes > limit) + { + self.0 = Vec::new(); + } + Ok(self.0.capacity()) + } + } + for component in 0..3 { + let build = || -> Result { + let mut frame = filter_frame([33, 33])?; + for ctu in &mut frame.filter_ctus { + ctu.sao[component] = Sao { + kind: 2, + offsets: [3, 1, -1, -3], + class: 0, + band: 0, + }; + } + Ok(frame) + }; + let mut cold = build()?; + let mut warm = build()?; + let live = cold.heap_bytes()? + cold.planes[component].len() * 2; + let mut idle = Idle(vec![0; 16]); + assert_eq!(cold.finalize(0, live)?, live); + assert_eq!(warm.finalize_reclaiming(0, live, Some(&mut idle))?, live); + assert_eq!(idle.0.capacity(), 0); + assert_eq!(warm.planes, cold.planes); + assert_eq!(warm.stats, cold.stats); + let mut cold = build()?; + let mut warm = build()?; + assert_eq!( + cold.finalize(0, live - 1), + Err(Error::Limit("reconstruction_memory")) + ); + assert_eq!( + warm.finalize_reclaiming(0, live - 1, Some(&mut idle)), + Err(Error::Limit("reconstruction_memory")) + ); + } + Ok(()) + } + #[test] fn sao_reads_deblocked_source_once_and_charges_its_live_copy() -> Result<()> { let mut frame = filter_frame([33, 33])?; diff --git a/crates/coreav-hevc/src/slice.rs b/crates/coreav-hevc/src/slice.rs index 332dc540a..c30d8e101 100644 --- a/crates/coreav-hevc/src/slice.rs +++ b/crates/coreav-hevc/src/slice.rs @@ -113,6 +113,10 @@ pub(crate) fn prefix(bits: &mut Bits<'_>, nal: NalHeader) -> Result { }) } +#[expect( + clippy::too_many_arguments, + reason = "header state and allocation boundary" +)] pub(crate) fn parse( bits: &mut Bits<'_>, nal: NalHeader, @@ -121,6 +125,7 @@ pub(crate) fn parse( p: &Pps, limits: &Limits, retain_entries: Option, + reserve_entries: &mut dyn FnMut(usize) -> Result<()>, ) -> Result { let mut entry_lengths = Vec::new(); let mut h = SliceHeader { @@ -376,9 +381,20 @@ pub(crate) fn parse( { return Err(Error::Limit("reconstruction_memory")); } + reserve_entries( + (h.entry_points as usize) + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; entry_lengths .try_reserve_exact(h.entry_points as usize) .map_err(|_| Error::Limit("entry_point_allocation"))?; + reserve_entries( + entry_lengths + .capacity() + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; } let width = bits.ue(31)? + 1; let mut total = 0u64; diff --git a/crates/coreav-hevc/src/tests.rs b/crates/coreav-hevc/src/tests.rs index 4e12f978f..c87c64362 100644 --- a/crates/coreav-hevc/src/tests.rs +++ b/crates/coreav-hevc/src/tests.rs @@ -873,6 +873,122 @@ fn selected_observer_failure_and_retained_scratch_limit_end_the_core() -> Result Ok(()) } +#[test] +fn reconstruction_scratch_reservations_cover_rbsp_allocation_and_actual_capacity() -> Result<()> { + struct Probe { + calls: Vec, + refuse: Option, + } + impl parser::PictureObserver for Probe { + fn reserve_scratch(&mut self, bytes: usize, _: &dpb::Dpb) -> Result<()> { + self.calls.push(bytes); + if self.refuse == Some(self.calls.len()) { + return Err(Error::Limit("fixture_scratch")); + } + Ok(()) + } + fn boundary(&mut self, _: &mut dpb::Dpb, _: usize) -> Result<()> { + Ok(()) + } + fn slice(&mut self, _: parser::SliceInput<'_>, _: &dpb::Dpb) -> Result<()> { + Ok(()) + } + fn sei(&mut self, _: SeiInfo, _: &dpb::Dpb, _: usize) -> Result<()> { + Ok(()) + } + fn output(&mut self, _: dpb::Emission<'_>, _: usize) -> Result<()> { + Ok(()) + } + } + let nal = &synthetic_parameter_nals()[0]; + let expected = [ + nal.len() - 2, + nal::rbsp(nal, &Limits::default())?.capacity(), + ]; + for refuse in [None, Some(1), Some(2)] { + let mut core = parser::StreamCore::new_reconstruction(Limits::default()); + let mut probe = Probe { + calls: Vec::new(), + refuse, + }; + let result = core.read_reconstructed(nal, usize::MAX, &mut probe); + if let Some(at) = refuse { + assert_eq!(result, Err(Error::Limit("fixture_scratch"))); + assert_eq!(probe.calls, expected[..at]); + assert_eq!( + core.read_reconstructed(nal, usize::MAX, &mut probe), + Err(Error::InvalidData("parser_failed")) + ); + } else { + result?; + assert_eq!(probe.calls, expected); + } + } + Ok(()) +} + +#[test] +fn entry_storage_reserves_requested_and_actual_capacity_before_slice_delivery() -> Result<()> { + let sets = two_ctu_parameter_nals(); + let sps = nal::rbsp(&sets[1], &Limits::default())?; + let sps = params::sps(&mut Bits::new(&sps), &Limits::default())?; + let pps = nal::rbsp(&sets[2], &Limits::default())?; + let mut pps = params::pps(&mut Bits::new(&pps), &Limits::default())?; + pps.wpp = true; + let mut w = params::tests::Writer::default(); + w.flag(true); + w.flag(false); + w.ue(0); + w.ue(2); + w.flag(true); + w.flag(false); + w.flag(false); + w.se(0); + w.flag(false); + w.ue(1); + w.ue(0); + w.u(1, 0); + let rbsp = padded(w); + let nal = NalHeader::parse(&[19 << 1, 1])?; + for refuse in [None, Some(1), Some(2)] { + let mut bits = Bits::new(&rbsp); + let prefix = slice::prefix(&mut bits, nal)?; + let mut calls = Vec::new(); + let result = slice::parse( + &mut bits, + nal, + prefix, + &sps, + &pps, + &Limits::default(), + Some(usize::MAX), + &mut |bytes| { + calls.push(bytes); + if refuse == Some(calls.len()) { + Err(Error::Limit("fixture_entries")) + } else { + Ok(()) + } + }, + ); + if let Some(at) = refuse { + assert!(matches!(result, Err(Error::Limit("fixture_entries")))); + assert_eq!(calls.len(), at); + } else { + let parsed = result?; + assert_eq!(parsed.entry_lengths, [1]); + assert_eq!( + calls, + [ + core::mem::size_of::(), + parsed.entry_lengths.capacity() * core::mem::size_of::() + ] + ); + } + } + Ok(()) +} + fn idr_segment(first: bool) -> Vec { let mut w = params::tests::Writer::default(); w.flag(first); @@ -1415,9 +1531,18 @@ fn equal_derived_long_term_references_do_not_hide_different_syntax() { let sps = params::sps(&mut Bits::new(&sps), &Limits::default()).unwrap(); let pps = nal::rbsp(&sets[2], &Limits::default()).unwrap(); let pps = params::pps(&mut Bits::new(&pps), &Limits::default()).unwrap(); - let second = slice::parse(&mut bits, nal, prefix, &sps, &pps, &Limits::default(), None) - .unwrap() - .header; + let second = slice::parse( + &mut bits, + nal, + prefix, + &sps, + &pps, + &Limits::default(), + None, + &mut |_| Ok(()), + ) + .unwrap() + .header; assert_eq!(first.long_term_count, second.long_term_count); assert_eq!(first.long_term, second.long_term); assert_eq!( diff --git a/docs/agents/ci.md b/docs/agents/ci.md index 5e695520c..1a19b43d4 100644 --- a/docs/agents/ci.md +++ b/docs/agents/ci.md @@ -76,6 +76,7 @@ List development caches with `docker volume ls --filter name=coreav-dev-`. After | A main push may reuse only a successful same-tree merge-group result of the same workflow | `.github/actions/merge-group-result` owns the lookup. `test_merge_group_result.py` rejects wrong trees, workflows, repositories, target branches, statuses and stale or unavailable evidence; `test_main_ci_plan.py` checks the retained main-only jobs and full-suite fallback | | AAC-enabled transcode CI runs the whole fixture test target, not a name-filtered subset that can omit source or schema regressions | `test_aac_source_build_runs_the_whole_transcode_target` reads the workflow's AAC-enabled invocation and rejects the former `opus_encode aac_encode` filters (`deny.yml`). Rust types cannot select tests in a shell command | | A CI description keeps a no-new-jobs claim attached to the change it describes | Not enforced: judgment | +| Explicit sanitizer-only HEVC dispatches run the sanitizer even when a reference input selector remains set | The sanitizer job owns the override; `test_explicit_sanitizer_mode_overrides_reference_selection` pins its condition. YAML scheduling cannot be enforced by decoder types | | Queue-detector evidence errors reset observation streaks, including truncated transfers and null API fields | `Api.request` validates declared Content-Length before parsing JSON and owns transfer-error classification. `test_content_length_premature_eof_cannot_become_an_empty_index`, `test_premature_eof_resets_an_existing_observation_streak` and the partial-read/null-field regressions in `scripts/tests/test_queue_detector.py` run in `deny.yml`. Runtime transfer completeness cannot be proved by a JSON value or a type | | A shadow queue-detector episode is reported once after revalidation without becoming a published alert | `report` owns the per-episode `shadowed_at` marker and shadow history, recording only revalidated outcomes. `test_shadow_episode_is_rechecked_once_but_can_still_publish_live` and the cancelled/later-failure regressions in `scripts/tests/test_queue_detector.py` run in `deny.yml`; bounded display history is not deduplication state | | Queue-detector failure diagnostics never echo bodies, tokens, exception arguments or arbitrary header/error fields; request metadata is conditional | `Api` owns a sanitized last-request trace, initially absent; `endpoint`, `response_facts`, `graphql_errors` and `diagnostic` emit closed labels, bounded digits and known query fields only. The `Diagnostics` regressions in `scripts/tests/test_queue_detector.py` inspect stdout, stderr, the job summary and persisted state with hostile bodies, headers and exception arguments; `test_raw_exception_arguments_are_never_rendered` requires a class-only diagnostic before request preparation. Types cannot classify untrusted HTTP strings | diff --git a/docs/agents/codecs.md b/docs/agents/codecs.md index dc040653e..ce9dcf7a6 100644 --- a/docs/agents/codecs.md +++ b/docs/agents/codecs.md @@ -8,6 +8,10 @@ H.264, AAC, Opus, VP9, AC-3, MP3 and AudioToolbox behavior; HEVC decoding and pl | H.264 decision unwind and ordinary errors have the same bounded result at one or multiple workers, including narrow and single-row geometry | Both scheduler paths catch unwind; `one_worker_and_parallel_paths_report_the_same_compute_failures` injects errors and panic at the beginning, middle and end for every effective worker shape. Runtime unwind and geometry cannot be encoded in the candidate type | | H.264 thread-start refusal preserves results from started workers and uses the caller if none starts | A single shared row loop owns both fallback and worker execution. `zero_and_partial_worker_start_failures_preserve_results_and_errors` injects errors through the real spawn-result branch and checks complete tables and decision-error selection; OS resource refusal needs runtime coverage | | H.264 neighbour reads require published wavefront dependencies rather than only thread-count identity | `unavailable_wavefront_neighbours_are_refused_before_border_reads` removes each required neighbour; `a_delayed_top_right_is_published_before_its_dependent_read` delays publication under concurrent rows. The eight-worker textured P round trip checks complete reconstructions through the independent decoder. Lowering the progress lead or omitting top-right availability must fail these checks; a candidate type does not establish spatial identity or publication order | +| Parallel HEVC decoding reserves a complete entropy-work bound before dispatch rather than overspending a shared budget and detecting it afterward | `Cabac::max_bins` bounds each row, and `decode_wpp` owns the sum reservation and serial budget-tail fallback. `substream_bound_covers_the_longest_mps_runs_and_truncated_reads` checks the arithmetic bound; `wavefront_workers_preserve_samples_work_and_failure_order` requires low-budget calls to stay serial. A racing counter cannot preserve the serial error order | +| HEVC worker reuse preserves cold admission, resets private state and accounts for idle allocations at each source allocation | Session ownership and scoped worker borrows prevent shared mutation and partial-state retention. `wavefront_warm_workers_match_cold_and_serial_across_picture_changes`, `wavefront_warm_memory_preserves_independent_cold_admission_boundaries` and `wavefront_warm_errors_panics_and_deadlines_release_every_private_frame` cover reset, cold limits and failure cleanup; omitted reset, release and transient-reservation mutations fail. The physical-envelope, empty-slot, output-capacity, RBSP, entry-storage and SAO allocation tests cover bookkeeping and release boundaries. Types cannot establish target padding, allocator capacity or a caller's runtime limit | +| HEVC WPP prediction reads published above-right samples rather than passing only PCM or all-serial thread-count comparisons | `wavefront_above_right_prediction_waits_for_publication` uses non-PCM mode-34 CUs, distinct upper samples and delayed upper-right publication; reducing the dependency lead or omitting later imports must fail. `threading_coverage` rejects all-serial complete threaded conformance runs. Runtime sample identity and scheduling require these checks rather than a static worker type | +| HEVC worker faults join all workers and distinguish decoder bugs from malformed data or caller limits | `wavefront_mid_row_errors_and_panics_join_every_worker` requires the serial error and bin count at 1/2/4/8 threads; `reconstruction_panics_are_backend_failures_not_media_limits` requires the existing backend failure kind. The standalone Apple adapter's exhaustive conversion and `decoder_internal_failure_is_not_a_metadata_or_resource_refusal` preserve that distinction outside the root workspace (`encoder-backends.yml`). `wavefront_deadline_expires_inside_a_parallel_dependency_wait` proves expiry after multiworker entry. The runtime fault location cannot be encoded in the worker type | | The H.264 encoder's SIMD exception covers AVX2 on x86-64 and NEON on AArch64 only; another instruction set needs a new policy record | Not mechanically enforced: review checks record 0044 point 2 in `docs/design/encoder-policy.md`. A capability check alone does not limit the policy's instruction-set scope, and Rust types cannot constrain policy prose | | The first NEON encoder implementation adds passing encoder verification on an arm64 CI runner before any NEON encoder kernel merges | Not enforced by CI today: record 0044 point 6 requires the first implementation to add arm64 encoder builds, Clippy and tests in both modes, complete-stream identity and differential fuzzing. Review blocks that implementation until its job exists and passes; the existing arm64 job covers the shared kernels and decoder, not the encoder | | H.264 quiet decisions use the coding path's component and prediction quantizer, and the selected chroma offset agrees with independent fixed arithmetic | `quiet_routes_match_fixed_division_and_coded_zero_levels` checks all QPs, actual coded zero levels and independent 5/12 chroma boundaries. It rejects both a luma quantizer in the chroma AC quiet path and a consistent inter chroma reversion to 2/12. The quantizer owns its arithmetic, but separate callers select the instance; a shared type or self-consistency check alone cannot prove that routing and policy | diff --git a/docs/design/hevc-threading.md b/docs/design/hevc-threading.md new file mode 100644 index 000000000..8f7b6ff79 --- /dev/null +++ b/docs/design/hevc-threading.md @@ -0,0 +1,49 @@ +# HEVC wavefront decoding + +## Choice + +Start with wavefront parallel processing (WPP) for a single-tile picture whose first independent slice supplies one entropy substream per CTU row. Keep every other slice on the existing serial path. Both paths use the same CTU decoder. This remains an opt-in `native-hevc` feature. + +Frame parallelism has the broadest eventual coverage, including streams without entry points. It also needs concurrent DPB lifetimes, reference-row publication after deblocking and SAO, motion-field publication, and cancellation across picture dependencies. The current DPB publishes only complete filtered pictures. Changing that contract is larger than a first threading path and overlaps the filtering work. + +Tiles have independent entropy and prediction state but help only streams that signal multiple tiles. Independent slices also need no entropy synchronization, but arbitrary slice boundaries and dependent segments complicate admission. Neither is a reliable assumption about the existing reference inputs. WPP already has parsed entry points and a serial second-CTU context checkpoint. It can expose most entropy and reconstruction work without changing the DPB or filtering. + +The expected gain is limited to WPP-signalled inputs and excludes serial filtering. Wide pictures offer more useful overlap than narrow pictures because each lower row trails the row above by two CTUs. The reference run must report actual eligible-picture coverage rather than treating non-WPP inputs as parallel work. No numerical speed-up is assumed before that run. + +## Ownership and ordering + +Each worker owns reconstruction scratch and its mutable picture storage. A row publishes completed CTUs, their prediction cells and SAO metadata under one mutex. A lower row waits until the upper row has completed the current column plus its upper-right neighbour, or the upper row has ended. It imports only published data. The second completed CTU also publishes the CABAC context checkpoint required by H.265 clause 9.3.2. Spatial prediction and QP derivation continue to use the existing slice and tile availability rules. + +Prediction imports only the last four luma rows of the upper band, including its last prediction-cell row. At column zero it also imports the first four rows, because CABAC restart availability checks the above-right CTU's first cell. Full CTUs remain published, and the final merge still imports every row for filtering and DPB storage. This avoids copying three complete upper CTUs at every column without changing worker ownership or memory admission. + +Rows are assigned in increasing order to a bounded group of scoped workers. Dependencies point only to the preceding row. Completed row storage is merged in row order before the unchanged whole-picture finalization and hash check. No partially reconstructed or filtered picture enters the DPB. The output callback still runs on the caller's thread in presentation order. + +The decoder session retains private worker allocations only after every row has reconciled successfully. Reuse requires the same worker count, geometry, bit depth, CTU shape, reference-storage shape, buffer lengths and total retained capacity. Every frame field and scratch buffer is reset from the new working picture before dispatch, with exhaustive destructuring requiring new fields to be handled. Workers borrow these private frames only within the thread scope. Failure drops them rather than returning partial state to the session. + +The codec exposes `PictureDecoder::with_threads`. The backend follows the existing native decoder option shape, `NativeOptions { threads: Option }`, with `None` selecting available host parallelism and explicit counts bounded to 1 through 16. Memory admission can reduce the effective count to one. Threadless targets retain serial decoding. The existing command-line decoder thread option supplies this backend option without adding a new CLI flag. + +## Failure and resource bounds + +Charge worker picture buffers, row publications, synchronization records and bounded thread stacks before starting work. Preserve the existing serial path when extra storage does not fit. Workers never call user callbacks or mutate the DPB. + +Select the worker count using the original cold calculation, including its transient scratch reservation, regardless of idle storage. Idle allocations are accounted separately and released before required allocations would exceed the same limit: RBSP and entry-point storage, working-picture and range storage, finalization scratch, and external output reservations. Requested and actual capacities are checked at the allocation boundary. An allocation/accounting hook at finalization's existing source allocation supplies this release point without changing filter arithmetic or control flow. Idle memory cannot turn an otherwise admitted decode into a refusal or admit additional workers. Changing the requested thread count or finishing the session releases the pool. + +Moving the active frame into a reserved slot keeps cache bookkeeping within the original fixed-state allowance. The allowance is still calculated from the legacy session layout, not by subtracting fields from the new layout, because padding is target-dependent. A physical-envelope test checks both session storage and active-frame metadata against that unchanged allowance on each supported test target. Idle worker-vector capacity is charged even when it contains no frames, and reported peaks include idle bytes. + +Reserve a conservative complete-substream bin bound for every row before dispatch. CABAC's normalized range is at most 510, and a decision or zero termination reduces it by at least two. At most 128 such bins can occur before another input bit is required. Bypass bins consume a bit each, and PCM restarts consume nine initialization bits. `128 * (8 * bytes + 1)` also covers a final failed read. Each row receives only its own bound. If their sum exceeds the remaining stream or picture budget, use serial decoding instead. This keeps actual entropy work within the caller's limit rather than discovering overspending after workers finish. + +For `B` entropy bytes across `R` rows, admission requires `1024 * B + 128 * R <= remaining_bins`. The default picture allowance is `2^28`, so even a fresh allowance limits parallel admission to approximately 256 KiB of entropy data; larger slices remain serial. Bounds are already sized per row, but all rows are reserved together. Reserving only active row batches and refunding unused work could admit larger slices. That requires a serial continuation point when the next batch cannot reserve its bound and is deferred rather than weakening the present work limit. + +Each row records its entropy consumption and outcome. Reconcile rows in bitstream order, so scheduling does not choose the reported syntax failure. A failed row wakes dependent rows. Every worker is joined before returning, including after a spawn failure, decode error or panic. A caught reconstruction panic returns `Error::Internal`, mapped to the backend's existing `Backend` failure kind, at every thread count. It is not malformed media or a caller resource limit. Deadline checks remain inside CTU work and dependency waits. Failure publishes no partial picture and the session remains failed. + +## Verification + +Compare complete visible output, coded-component hashes, picture order, timestamps and work counters with one thread at 2, 4 and 8 threads. Run the admitted conformance set and generated WPP boundary cases, including one-column rows, partial final rows, malformed entry points, dependent-segment continuation, budget exhaustion, callback failure and deadline expiry. Stress repeated decoding under a hard process timeout. Exercise the synchronization owner with ThreadSanitizer on Linux where supported. Keep unsupported sanitizer coverage explicit. + +The generated above-right fixture has three CTUs per row and two rows. Its lower row contains non-PCM angular-mode-34 CUs with zero residual. Different upper-row samples make the lower row's top-right CU samples depend on the above-right CTU. A test-only hook delays publication of upper column two until lower column one is waiting; the test rejects early lower publication and compares every plane and work counter with serial decoding. Separate hooks inject a middle-row error, panic and deadline expiry with real multiworker entry. Complete conformance runs with `--threads` greater than one fail if every picture falls back to serial decoding. + +Warm-worker tests compare serial, cold and reused output across changed samples, picture indices, geometry and bit depth. Independent cold decodes locate constructor and worker-selection boundaries, including large input-scratch reservations; reused decodes must retain the same success, refusal and worker counts at each boundary. Removing the reset, idle release, or cold transient reservation must fail these tests. Parser callback tests require both requested and actual capacity reservations and propagate refusal before slice delivery. + +Measure the existing reference inputs on GitHub Actions with full candidate and main SHAs, separate executables, warm-up and alternating repeated calls at automatic, 1, 4, 8 and 16 threads. The reference helper takes its memory budget from the backend policy defaults and records both that budget and the resolved automatic worker limit. First verify pixels against the qualified pinned oracle. Export enhancement metadata rather than applying it so both arms reconstruct the HEVC base raster, without resizing or pixel conversion. Report main time divided by candidate time, candidate one-thread time divided by candidate multithread time, CPU usage, host core count and WPP coverage. Separate WPP from non-WPP inputs when checking the automatic-worker cost. These are shared-runner reference figures, not speed acceptance bars. Retain the result privately and delete the disposable runs afterward. + +No change in this work modifies `inverse_residual`, filter arithmetic or control flow, the default build, decoder isolation, or the admitted coding-tool set. Sources are ITU-T H.265 and conformance data only. diff --git a/scripts/hevc_conformance.py b/scripts/hevc_conformance.py index 29a2e055b..953b227ed 100644 --- a/scripts/hevc_conformance.py +++ b/scripts/hevc_conformance.py @@ -1235,7 +1235,7 @@ def corpus_shard(files, index, count): return sorted(files, key=lambda item: (-item['size'], item['sha256']))[index::count] -def fetch_corpus(args, inspect=False): +def fetch_corpus(args, inspect=False, inspector=None): require(sys.platform == 'linux', 'hostile corpus files require Linux') sys.path.insert(0, str(ROOT / 'tools/compat-corpus')) import round_files @@ -1307,7 +1307,7 @@ def fetch_corpus(args, inspect=False): if getattr(args, 'boundary_native', None): require(complete is not None and args.boundary_native.is_file(), 'boundary_requires_packet_native') complete['boundary'] = args.boundary_native - records[index] = corpus_record(item, path, args.out, qualified, intra=options, complete=complete) + records[index] = (inspector or corpus_record)(item, path, args.out, qualified, intra=options, complete=complete) write_json(args.out / 'corpus-census.json', corpus_result(records, expected, scope)) finally: path.unlink(missing_ok=True) diff --git a/scripts/hevc_decode.py b/scripts/hevc_decode.py index d1bafdc33..3637bd824 100644 --- a/scripts/hevc_decode.py +++ b/scripts/hevc_decode.py @@ -230,7 +230,8 @@ def streams(args, census): def command(args, source, *, memory=None, stream_bins=None, picture_bins=None): return [args.native, source, str(args.max_memory_bytes if memory is None else memory), str(args.max_stream_bins if stream_bins is None else stream_bins), - str(args.max_picture_bins if picture_bins is None else picture_bins)] + str(args.max_picture_bins if picture_bins is None else picture_bins), + *([str(args.threads)] if getattr(args, 'threads', None) is not None else [])] def controls(args, stream, directory, expected): @@ -262,6 +263,24 @@ def controls(args, stream, directory, expected): truncated.unlink(missing_ok=True) +def threading_coverage(entries, threads): + parallel_pictures = 0 + max_workers = 0 + for entry in entries: + if not entry['complete']: + continue + summary = entry['summary'] + pictures, workers = summary.get('parallel_pictures'), summary.get('max_workers') + require(harness.integer(pictures) and pictures <= summary['decoded_pictures'] + and harness.integer(workers) and workers <= min(threads, 16), 'native_thread_counters') + require(pictures == 0 or workers > 1 and summary['stats']['wpp_restarts'] > 0, + 'native_parallel_wpp_work') + parallel_pictures += pictures + max_workers = max(max_workers, workers) + require(threads == 1 or parallel_pictures > 0, 'native_parallel_coverage_missing') + return dict(parallel_pictures=parallel_pictures, max_workers=max_workers) + + def run(args): require(sys.platform == 'linux', 'linux_actions_required') require(0 < args.max_memory_bytes <= 4 << 30 @@ -283,7 +302,7 @@ def run(args): max_picture_bins=args.max_picture_bins) dispositions = [dict(test=case['test'], sha256=stream['sha256'], status=stream['status']) for case in census['archives'] for stream in case['streams']] - result = dict(complete=False, scope='diagnostic_subset' if args.select_test else 'complete_c1', + result = dict(complete=False, passed=False, scope='diagnostic_subset' if args.select_test else 'complete_c1', identities=identities, budgets=budgets, denominator=denominator, results=[], controls=[]) result['source_dispositions'] = dispositions controlled = set() @@ -346,6 +365,8 @@ def run(args): and harness.digest(args.data / 'acquisition.json') == identities['acquisition'] and all(harness.digest(getattr(qualified, name)) == value for name, value in identities['oracle'].items()), 'run_identity_changed') + if getattr(args, 'threads', None) is not None and not args.select_test: + result['threading'] = threading_coverage(result['results'], args.threads) result['passed'] = bool(result['results']) and all(row['complete'] for row in result['results']) result['complete'] = result['passed'] and not args.select_test and len(result['results']) == len(denominator) result['output_pictures'] = sum(row.get('summary', {}).get('output_pictures', 0) for row in result['results']) @@ -685,6 +706,7 @@ def main(): parser.add_argument('--max-memory-bytes', type=int, default=1 << 30) parser.add_argument('--max-stream-bins', type=int, default=1 << 40) parser.add_argument('--max-picture-bins', type=int, default=1 << 28) + parser.add_argument('--threads', type=int, choices=range(1, 17)) parser.add_argument('--select-test', action='append', default=[]) args = parser.parse_args() try: diff --git a/scripts/hevc_reference.py b/scripts/hevc_reference.py index aaf890193..dd2f8d6dc 100644 --- a/scripts/hevc_reference.py +++ b/scripts/hevc_reference.py @@ -48,13 +48,14 @@ def command(args, timeout=180): def ffmpeg_command(binary, path, threads, output): return [binary, '-hide_banner', '-loglevel', 'error', '-nostdin', '-noautorotate', - '-flags', 'unaligned', '-c:v', 'hevc', *(['-threads', '1'] if threads else []), + '-flags', 'unaligned', '-c:v', 'hevc', '-export_side_data', '+enhancements', + *(['-threads', '1'] if threads else []), '-i', path, '-map', '0:v:0', '-an', '-sn', '-dn', '-noautoscale', '-fps_mode', 'passthrough', *output] -def verify(native, oracle, source): - raw, _ = command([native, source, 'check']) +def verify(native, oracle, source, *, timeout=180): + raw, _ = command([native, source, 'check'], timeout=timeout) rows = [json.loads(line) for line in raw.splitlines()] pictures = [row for row in rows if row.get('type') == 'picture'] if not pictures or rows[-1].get('complete') is not True: @@ -64,7 +65,7 @@ def verify(native, oracle, source): (depth, pictures[0]['width'], pictures[0]['height']) for p in pictures): raise ValueError('depth_change') metadata, _ = command([oracle.ffprobe, '-v', 'error', - '-select_streams', 'v:0', '-show_entries', 'stream=pix_fmt,width,height', '-of', 'json', source]) + '-select_streams', 'v:0', '-show_entries', 'stream=pix_fmt,width,height', '-of', 'json', source], timeout=timeout) streams = json.loads(metadata)['streams'] if len(streams) != 1 or (streams[0]['width'], streams[0]['height']) != (pictures[0]['width'], pictures[0]['height']): raise ValueError('oracle_geometry') @@ -72,7 +73,7 @@ def verify(native, oracle, source): if pixel_format not in ({'yuv420p', 'yuvj420p'} if depth == 8 else {'yuv420p10le'}): raise ValueError('oracle_pixel_format') reference, _ = command(ffmpeg_command(oracle.ffmpeg, source, True, - ['-pix_fmt', '+' + pixel_format, '-f', 'framemd5', '-'])) + ['-pix_fmt', '+' + pixel_format, '-f', 'framemd5', '-']), timeout=timeout) dimensions = re.findall(rb'^#dimensions\s+0:\s+(\d+)x(\d+)\s*$', reference, re.M) if dimensions != [(str(pictures[0]['width']).encode(), str(pictures[0]['height']).encode())]: raise ValueError('oracle_geometry') diff --git a/scripts/hevc_threading.py b/scripts/hevc_threading.py new file mode 100644 index 000000000..ebc533434 --- /dev/null +++ b/scripts/hevc_threading.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Paired HEVC threading reference with complete-output identity checks.""" + +import argparse +import json +import os +import pathlib +import platform +import re +import statistics +import subprocess +import sys +import tempfile + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import hevc_reference as reference +import hevc_conformance as conformance +import oracle_pin +import round_files +import test_oracle + +THREADS = (0, 1, 4, 8, 16) + + +def syntax_record(item, source, directory, oracle, parser): + directory = directory / item['sha256'] + elementary = directory / 'syntax.hevc' + record = dict(item, status='blocked') + try: + code, _, _ = conformance.child([oracle.ffmpeg, '-nostdin', '-v', 'error', '-i', source, + '-map', '0:v:0', '-c:v', 'copy', '-bsf:v', 'hevc_mp4toannexb', '-f', 'hevc', '-y', elementary], + directory, 'extract', file_limit=4 << 30) + conformance.require(code == 0 and 0 < elementary.stat().st_size < 4 << 30, 'syntax_extraction_failed') + code, stdout, _ = conformance.child([parser, elementary], directory, 'headers') + conformance.require(code == 0, 'syntax_inspection_failed') + rows = [json.loads(line) for line in stdout.read_text().splitlines()] + conformance.require(rows and rows[-1].get('kind') == 'summary' and rows[-1].get('complete') is True, + 'syntax_inspection_incomplete') + tools = rows[-1]['tools'] + conformance.require(all(type(tools.get(key)) is bool for key in ('wpp', 'tiles')), 'syntax_tools_missing') + record.update(status='inspected', wpp=tools['wpp'], tiles=tools['tiles'], parser_sha256=conformance.digest(parser)) + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError) as error: + record['failure'] = type(error).__name__ + finally: + elementary.unlink(missing_ok=True) + return record + + +def census(argv): + parser = argparse.ArgumentParser(description='Count active WPP and tile syntax over the complete existing roster') + for name in ('parser', 'oracle-bin', 'dest', 'out'): + parser.add_argument('--' + name, required=True, type=pathlib.Path) + args = parser.parse_args(argv) + args.manifests = None + args.native = None + def inspect(item, source, directory, oracle, **unused): + return syntax_record(item, source, directory, oracle, args.parser) + try: + conformance.fetch_corpus(args, inspect=True, inspector=inspect) + finally: + path = args.out / 'corpus-census.json' + if path.exists(): + report = json.loads(path.read_text()) + files = [row for row in report['files'] if row['status'] == 'inspected'] + counts = dict(files=len(report['files']), inspected=len(files), + wpp=sum(row['wpp'] for row in files), tiles=sum(row['tiles'] for row in files), + neither=sum(not row['wpp'] and not row['tiles'] for row in files), + both=sum(row['wpp'] and row['tiles'] for row in files), complete=report['complete']) + (args.out / 'tool-counts.json').write_text(json.dumps(counts, indent=2) + '\n') + return 0 + + +def normalized(value): + if isinstance(value, dict): + return {key: normalized(item) for key, item in value.items() if key not in {'memory_bytes', 'parallel_pictures', 'max_workers'}} + if isinstance(value, list): + return [normalized(item) for item in value] + return value + + +def signature(binary, source, threads): + stdout, _ = reference.command([binary, source, 'check', threads], timeout=600) + rows = [json.loads(line) for line in stdout.splitlines()] + if not rows or rows[-1].get('complete') is not True: + raise ValueError('incomplete_native') + return normalized(rows) + + +def summarize(files): + result = {} + for depth in (8, 10): + selected = [row for row in files if row.get('status') == 'timed' and row['depth'] == depth] + ratios = {str(threads): [] for threads in THREADS} + scaling = {str(threads): [] for threads in THREADS} + oracle = {str(threads): [] for threads in THREADS} + for row in selected: + medians = {arm: statistics.median(call['wall'] for call in row['calls'] if call['arm'] == arm) + for arm in {call['arm'] for call in row['calls']}} + for threads in THREADS: + ratios[str(threads)].append(medians[f'main_{threads}'] / medians[f'candidate_{threads}']) + scaling[str(threads)].append(medians['candidate_1'] / medians[f'candidate_{threads}']) + oracle[str(threads)].append(medians[f'candidate_{threads}'] / medians['ffmpeg_one']) + result[str(depth)] = dict(files=len(selected), main_over_candidate={key: statistics.median(values) if values else None for key, values in ratios.items()}, + candidate_one_over_threads={key: statistics.median(values) if values else None for key, values in scaling.items()}, + candidate_over_ffmpeg_one={key: statistics.median(values) if values else None for key, values in oracle.items()}) + return result + + +def measure(args, oracle, source, identity): + row = dict(id=identity, input_sha256=conformance.digest(source), status='checking_main') + try: + row.update(reference.verify(args.main, oracle, source, timeout=600)) + except (ValueError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + row.update(status='baseline_not_verified', failure=type(error).__name__) + if isinstance(error, ValueError) and str(error) in { + 'incomplete_native', 'depth_change', 'oracle_geometry', 'oracle_pixel_format', 'pixel_mismatch'}: + row['verification_failure'] = str(error) + if isinstance(error, subprocess.TimeoutExpired): + program = error.cmd[0] + row.update(timeout_seconds=error.timeout, + verification_step='baseline_decode' if program == str(args.main) + else 'oracle_probe' if program == str(oracle.ffprobe) else 'oracle_decode') + return row + expected = signature(args.main, source, 1) + row['status'] = 'checking_candidate' + for threads in (0, 1, 2, 4, 8, 16): + if signature(args.native, source, threads) != expected: + raise ValueError('thread_output_mismatch') + commands = {f'{arm}_{threads}': [binary, source, 'decode', threads] + for arm, binary in [('main', args.main), ('candidate', args.native)] for threads in THREADS} + commands['ffmpeg_one'] = reference.ffmpeg_command(oracle.ffmpeg, source, True, ['-progress', 'pipe:1', '-f', 'null', '-']) + commands['ffmpeg_default'] = reference.ffmpeg_command(oracle.ffmpeg, source, False, ['-progress', 'pipe:1', '-f', 'null', '-']) + names = tuple(commands) + work = None + calls = [] + for repeat in range(-1, args.repeats): + offset = max(0, repeat) % len(names) + order = names[offset:] + names[:offset] + if repeat % 2: + order = tuple(reversed(order)) + for name in order: + stdout, receipt = reference.command(commands[name], timeout=600) + reference.check_work('coreav' if name.startswith(('main_', 'candidate_')) else name, stdout, row['frames']) + if name.startswith(('main_', 'candidate_')): + data = json.loads(stdout) + observed = {key: data[key] for key in ('pictures', 'decoded_pictures', 'bins')} + if work is not None and work != observed: + raise ValueError('thread_work_mismatch') + work = observed + receipt.update(thread_limit=data['thread_limit'], max_memory_bytes=data['max_memory_bytes']) + if name.startswith('candidate_'): + receipt.update(parallel_pictures=data['parallel_pictures'], max_workers=data['max_workers']) + calls.append(dict(arm=name, repeat=repeat, **receipt)) + row.update(status='timed', calls=[call for call in calls if call['repeat'] >= 0], warmups=[call for call in calls if call['repeat'] < 0], work=work) + return row + + +def main(): + if sys.argv[1:2] == ['census']: + return census(sys.argv[2:]) + parser = argparse.ArgumentParser(description=__doc__) + for name in ('native', 'main', 'oracle-bin', 'out'): + parser.add_argument('--' + name, required=True, type=pathlib.Path) + for name in ('candidate', 'baseline'): + parser.add_argument('--' + name, required=True) + parser.add_argument('--shard-index', type=int, default=0) + parser.add_argument('--shard-count', type=int, default=1) + parser.add_argument('--only', default='') + parser.add_argument('--repeats', type=int, choices=range(3, 10), default=5) + parser.add_argument('--frames', type=int, choices=range(1, 65), default=32) + args = parser.parse_args() + if any(re.fullmatch('[0-9a-f]{40}', getattr(args, name)) is None for name in ('candidate', 'baseline')): + parser.error('full_arm_shas_required') + if not 0 <= args.shard_index < args.shard_count <= 30: + parser.error('invalid_shard') + ids = [f'd{index:02}' for index in range(26)] + [f'c{index:02}' for index in range(4)] + selected = set(args.only.split(',')) if args.only else set(ids) + if not selected <= set(ids): + parser.error('unknown_input_identity') + args.only = {identity for index, identity in enumerate(ids) + if index % args.shard_count == args.shard_index and identity in selected} + if not args.only: + parser.error('empty_shard') + oracle = test_oracle.require(args.oracle_bin / 'ffmpeg', args.oracle_bin / 'ffprobe') + args.out.mkdir(parents=True, exist_ok=True) + report = dict(candidate=args.candidate, baseline=args.baseline, candidate_sha256=conformance.digest(args.native), + baseline_sha256=conformance.digest(args.main), oracle=oracle_pin.load()['BUILD'], + host=dict(cpu=next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), + cores=os.cpu_count(), affinity=sorted(os.sched_getaffinity(0)), kernel=platform.release()), + scope='whole-process Annex B decode-to-null; startup, framing and teardown included; count 0 selects automatic workers with the backend memory budget; main is serial at every requested count', + selected_ids=sorted(args.only), frames_cap=args.frames, repeats=args.repeats, complete=False, passed=True, files=[]) + pool = round_files.pool() + pool.limits.max_requests = 4000 + pool.limits.max_bytes = 32 << 30 + try: + with tempfile.TemporaryDirectory(prefix='hevc-threading-') as temp: + for identity, source, original_sha in reference.inputs(args, oracle, pathlib.Path(temp), pool): + try: + row = measure(args, oracle, source, identity) + if row['status'] != 'timed': + report['passed'] = False + except (ValueError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + row = dict(id=identity, status='failed', failure=type(error).__name__) + if isinstance(error, ValueError): + row['failure'] = str(error) + report['passed'] = False + row['source_sha256'] = original_sha + report['files'].append(row) + (args.out / 'results.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(dict(id=identity, status=row['status'], failure=row.get('failure'))), flush=True) + if not report['passed']: + break + report['complete'] = {row['id'] for row in report['files']} == args.only + finally: + pool.close() + report['r2_requests'] = pool.limits.requests + report['summary'] = summarize(report['files']) + (args.out / 'results.json').write_text(json.dumps(report, indent=2) + '\n') + return 0 if report['complete'] and report['passed'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/test_hevc_threading.py b/scripts/tests/test_hevc_threading.py new file mode 100644 index 000000000..00336b2e5 --- /dev/null +++ b/scripts/tests/test_hevc_threading.py @@ -0,0 +1,157 @@ +import copy +import json +import pathlib +import subprocess +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +import hevc_threading as threading +import hevc_decode + + +class ThreadingReference(unittest.TestCase): + def test_explicit_sanitizer_mode_overrides_reference_selection(self): + workflow = pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml' + job = workflow.read_text().split(' thread-sanitizer:', 1)[1] + self.assertIn("if: ${{ inputs.sanitizer_only || inputs.only == '' }}", job) + + def test_threaded_conformance_rejects_silent_serial_fallback(self): + serial = dict(complete=True, summary=dict(decoded_pictures=3, parallel_pictures=0, max_workers=1, + stats=dict(wpp_restarts=2))) + self.assertEqual(hevc_decode.threading_coverage([serial], 1), dict(parallel_pictures=0, max_workers=1)) + for threads in (2, 4, 8): + with self.assertRaisesRegex(ValueError, 'native_parallel_coverage_missing'): + hevc_decode.threading_coverage([serial], threads) + parallel = copy.deepcopy(serial) + parallel['summary'].update(parallel_pictures=2, max_workers=2) + self.assertEqual(hevc_decode.threading_coverage([serial, parallel], 4), + dict(parallel_pictures=2, max_workers=2)) + parallel['summary']['stats']['wpp_restarts'] = 0 + with self.assertRaisesRegex(ValueError, 'native_parallel_wpp_work'): + hevc_decode.threading_coverage([parallel], 4) + + def test_unverified_baseline_cannot_pass_or_continue_the_shard(self): + with tempfile.TemporaryDirectory() as temp: + out = pathlib.Path(temp) / 'out' + args = ['driver', '--native', '/native', '--main', '/main', '--oracle-bin', '/oracle', '--out', str(out), + '--candidate', 'a' * 40, '--baseline', 'b' * 40, '--only', 'd00,d01'] + pool = types.SimpleNamespace(limits=types.SimpleNamespace(requests=0), close=lambda: None) + with patch.object(sys, 'argv', args), \ + patch.object(threading.test_oracle, 'require', return_value=object()), \ + patch.object(threading.conformance, 'digest', return_value='c' * 64), \ + patch.object(threading.oracle_pin, 'load', return_value=dict(BUILD='qualified')), \ + patch.object(pathlib.Path, 'read_text', return_value='model name : fixture host\n'), \ + patch.object(threading.os, 'sched_getaffinity', return_value={0, 1}, create=True), \ + patch.object(threading.round_files, 'pool', return_value=pool), \ + patch.object(threading.reference, 'inputs', return_value=[('d00', pathlib.Path('/input'), 'd' * 64), + ('d01', pathlib.Path('/next'), 'e' * 64)]), \ + patch.object(threading, 'measure', return_value=dict(id='d00', status='baseline_not_verified')) as measure: + self.assertEqual(threading.main(), 1) + measure.assert_called_once() + report = json.loads((out / 'results.json').read_text()) + self.assertFalse(report['passed']) + self.assertEqual(report['selected_ids'], ['d00', 'd01']) + + def test_baseline_failure_retains_only_closed_verification_reasons(self): + args = types.SimpleNamespace(native='/candidate', main=pathlib.Path('/main')) + for reason in ('incomplete_native', 'depth_change', 'oracle_geometry', 'oracle_pixel_format', + 'pixel_mismatch', 'untrusted diagnostic'): + with self.subTest(reason=reason), \ + patch.object(threading.conformance, 'digest', return_value='a' * 64), \ + patch.object(threading.reference, 'verify', side_effect=ValueError(reason)): + row = threading.measure(args, object(), pathlib.Path('/input'), 'd00') + self.assertEqual(row['status'], 'baseline_not_verified') + self.assertEqual(row['failure'], 'ValueError') + self.assertEqual(row.get('verification_failure'), None if reason == 'untrusted diagnostic' else reason) + + def test_baseline_timeout_records_the_failing_verification_step(self): + args = types.SimpleNamespace(native='/candidate', main=pathlib.Path('/main')) + oracle = types.SimpleNamespace(ffmpeg='/qualified-ffmpeg', ffprobe='/qualified-ffprobe') + for program, step in [('/main', 'baseline_decode'), (oracle.ffprobe, 'oracle_probe'), + (oracle.ffmpeg, 'oracle_decode')]: + with patch.object(threading.conformance, 'digest', return_value='a' * 64), \ + patch.object(threading.reference, 'verify', side_effect=subprocess.TimeoutExpired([program], 600)) as verify: + row = threading.measure(args, oracle, pathlib.Path('/input'), 'd00') + self.assertEqual((row['status'], row['verification_step'], row['timeout_seconds']), + ('baseline_not_verified', step, 600)) + verify.assert_called_once_with(args.main, oracle, pathlib.Path('/input'), timeout=600) + + def test_census_counts_only_complete_native_syntax_and_uses_qualified_extraction(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + source = root / 'source' + source.write_bytes(b'fixture') + oracle = types.SimpleNamespace(ffmpeg='/qualified-ffmpeg') + commands = [] + def child(command, directory, label, **unused): + commands.append(command) + directory.mkdir(parents=True, exist_ok=True) + if label == 'extract': + pathlib.Path(command[-1]).write_bytes(b'fixture') + stdout = directory / (label + '.stdout') + stdout.write_text('{"kind":"summary","complete":true,"tools":{"wpp":true,"tiles":false}}') + return 0, stdout, directory / 'stderr' + with patch.object(threading.conformance, 'child', side_effect=child), \ + patch.object(threading.conformance, 'digest', return_value='b' * 64): + result = threading.syntax_record(dict(sha256='a' * 64), source, root, oracle, '/native-parser') + self.assertEqual((result['status'], result['wpp'], result['tiles']), ('inspected', True, False)) + self.assertEqual(commands[0][0], oracle.ffmpeg) + self.assertEqual(commands[1][0], '/native-parser') + self.assertFalse((root / ('a' * 64) / 'syntax.hevc').exists()) + + def test_identity_ignores_only_resource_and_scheduling_counters(self): + one = dict(output_md5='a' * 32, stats=dict(ctus=3, memory_bytes=100), parallel_pictures=0, max_workers=1) + many = copy.deepcopy(one) + many.update(parallel_pictures=1, max_workers=8) + many['stats']['memory_bytes'] = 500 + self.assertEqual(threading.normalized(one), threading.normalized(many)) + for owner, key in [(many, 'output_md5'), (many['stats'], 'ctus')]: + old = owner[key] + owner[key] = 'changed' + self.assertNotEqual(threading.normalized(one), threading.normalized(many)) + owner[key] = old + + def test_thread_count_reaches_every_conformance_control(self): + args = types.SimpleNamespace(native='/native', max_memory_bytes=100, max_stream_bins=200, max_picture_bins=300, threads=8) + self.assertEqual(hevc_decode.command(args, '/input'), ['/native', '/input', '100', '200', '300', '8']) + self.assertEqual(hevc_decode.command(args, '/input', stream_bins=0), ['/native', '/input', '100', '0', '300', '8']) + del args.threads + self.assertEqual(hevc_decode.command(args, '/input'), ['/native', '/input', '100', '200', '300']) + + def test_signature_requires_completion_and_supplies_threads(self): + with patch.object(threading.reference, 'command', return_value=(b'{"type":"summary","complete":true}', {})) as child: + for threads in (0, 4): + threading.signature('/native', '/input', threads) + self.assertEqual(child.call_args.args[0], ['/native', '/input', 'check', threads]) + with patch.object(threading.reference, 'command', return_value=(b'{"complete":false}', {})): + with self.assertRaisesRegex(ValueError, 'incomplete_native'): + threading.signature('/native', '/input', 4) + + def test_output_mismatch_prevents_any_timing(self): + args = types.SimpleNamespace(native='/candidate', main='/main') + with patch.object(threading.conformance, 'digest', return_value='a' * 64), \ + patch.object(threading.reference, 'verify', return_value=dict(frames=2)), \ + patch.object(threading, 'signature', side_effect=[[dict(output_md5='a')], [dict(output_md5='b')]]), \ + patch.object(threading.reference, 'command') as child: + with self.assertRaisesRegex(ValueError, 'thread_output_mismatch'): + threading.measure(args, object(), pathlib.Path('/input'), 'd00') + child.assert_not_called() + + def test_ratio_direction_and_every_requested_count(self): + calls = [] + for threads in threading.THREADS: + calls += [dict(arm=f'main_{threads}', wall=12), dict(arm=f'candidate_{threads}', wall=6 / (threads or 4))] + calls += [dict(arm='ffmpeg_one', wall=2)] + summary = threading.summarize([dict(status='timed', depth=8, calls=calls)])['8'] + self.assertEqual(summary['main_over_candidate']['4'], 8) + self.assertEqual(summary['main_over_candidate']['0'], 8) + self.assertEqual(summary['candidate_one_over_threads']['4'], 4) + self.assertEqual(summary['candidate_over_ffmpeg_one']['1'], 3) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_reference_hevc.py b/scripts/tests/test_reference_hevc.py index 1b6915c09..177985b8d 100644 --- a/scripts/tests/test_reference_hevc.py +++ b/scripts/tests/test_reference_hevc.py @@ -55,6 +55,11 @@ def test_candidate_must_be_on_a_branch_before_it_builds_or_sees_credentials(self self.assertLess(admission, workflow.index('secrets.')) self.assertIn('test -n "$(git branch -r --contains "$CANDIDATE")"', workflow[admission:workflow.index('cargo build')]) + def test_dispatch_frame_count_is_numeric_at_the_reusable_boundary(self): + workflow = (SCRIPTS.parent / '.github/workflows/hevc-reference.yml').read_text() + threading = workflow[workflow.index(' threading:'):] + self.assertIn('frames: ${{ fromJSON(inputs.frames) }}', threading) + def test_diagnostic_mode_never_times_verified_files(self): args = type('Args', (), dict(native=pathlib.Path('/native'), oracle_bin=pathlib.Path('/oracle'), diagnose_only=True))() with tempfile.TemporaryDirectory() as temp: @@ -173,9 +178,10 @@ def test_verify_compares_bytes_hashes_and_complete_frame_count(self): (b'0, 0, 0, 1, 10, abcd\n', False), (b'', False)]: oracle = b'#dimensions 0: 2x2\n' + oracle metadata = json.dumps(dict(streams=[dict(pix_fmt='yuv420p10le', width=2, height=2)])).encode() - with self.subTest(oracle=oracle), patch.object(reference, 'command', side_effect=[(native, {}), (metadata, {}), (oracle, {})]): + with self.subTest(oracle=oracle), patch.object(reference, 'command', side_effect=[(native, {}), (metadata, {}), (oracle, {})]) as command: if passes: - self.assertEqual(reference.verify('/native', ORACLE, '/input')['frames'], 1) + self.assertEqual(reference.verify('/native', ORACLE, '/input', timeout=600)['frames'], 1) + self.assertEqual([call.kwargs['timeout'] for call in command.call_args_list], [600, 600, 600]) else: with self.assertRaisesRegex(ValueError, 'pixel_mismatch'): reference.verify('/native', ORACLE, '/input') @@ -207,6 +213,27 @@ def test_full_range_uses_unchanged_oracle_format_and_wrong_depth_is_refused(self reference.verify('/native', ORACLE, '/input') self.assertEqual(child.call_count, 2) + def test_base_raster_verification_exports_enhancements_in_every_oracle_arm(self): + picture = dict(type='picture', bit_depth=8, output_bytes=6, output_md5='base', width=2, height=2) + native = ('\n'.join(json.dumps(row) for row in + (picture, dict(type='summary', complete=True, output_md5='base')))).encode() + def child(command, **unused): + if command[0] == '/native': + return native, {} + if command[0] == ORACLE.ffprobe: + return b'{"streams":[{"pix_fmt":"yuv420p","width":2,"height":2}]}', {} + if '-export_side_data' in command: + self.assertEqual(command[command.index('-export_side_data') + 1], '+enhancements') + self.assertLess(command.index('-export_side_data'), command.index('-i')) + return b'#dimensions 0: 2x2\n0, 0, 0, 1, 6, base\n', {} + return b'#dimensions 0: 4x2\n0, 0, 0, 1, 12, enhanced\n', {} + with patch.object(reference, 'command', side_effect=child): + self.assertEqual(reference.verify('/native', ORACLE, '/input')['frames'], 1) + for single in (True, False): + command = reference.ffmpeg_command(ORACLE.ffmpeg, '/input', single, ['-f', 'null', '-']) + self.assertIn('-export_side_data', command) + self.assertEqual(command[command.index('-export_side_data') + 1], '+enhancements') + def test_oracle_arms_select_hevc_with_correct_thread_scope(self): one = reference.ffmpeg_command('ffmpeg', 'input', True, ['-f', 'null', '-']) default = reference.ffmpeg_command('ffmpeg', 'input', False, ['-f', 'null', '-'])