Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions .github/workflows/h264-encoder.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ on:
branches: [main]
paths:
- crates/coreav-h264-encoder/**
- crates/coreav-cli/**
- crates/coreav-transcode/**
- crates/coreav-frame/**
- scripts/native_h264_cli.py
- crates/coreav-kernels-h264/**
- crates/coreav-h264/**
- scripts/h264_g1*
Expand All @@ -30,6 +34,10 @@ on:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- crates/coreav-h264-encoder/**
- crates/coreav-cli/**
- crates/coreav-transcode/**
- crates/coreav-frame/**
- scripts/native_h264_cli.py
- crates/coreav-kernels-h264/**
- crates/coreav-h264/**
- scripts/h264_g1*
Expand Down Expand Up @@ -59,6 +67,87 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
native-h264-cli:
runs-on: ubuntu-24.04
timeout-minutes: 30
defaults:
run:
working-directory: candidate
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: candidate
persist-credentials: false
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: baseline
ref: ${{ github.event.pull_request.base.sha || github.sha }}
persist-credentials: false
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
toolchain: stable
components: clippy
- name: Preserve base-build byte references
run: |
cd ../baseline
cargo build --locked --release -p coreav-cli
cp target/release/coreav "$RUNNER_TEMP/default-base"
cargo build --locked --release -p coreav-cli --features native-h264
cp target/release/coreav "$RUNNER_TEMP/external-base"
- name: Keep native encoding out of default and release builds
run: |
set -euo pipefail
for features in '' release-default release-full; do
cargo tree --locked -p coreav-cli --features "$features" -e normal > "$RUNNER_TEMP/native-tree"
if grep -q coreav-h264-encoder "$RUNNER_TEMP/native-tree"; then
echo 'native H.264 encoder entered a default or release feature set' >&2
exit 1
fi
done
cargo build --locked --release -p coreav-cli
cp target/release/coreav "$RUNNER_TEMP/default-candidate"
- name: Check opt-in adapters and existing CLI limits
run: |
python3 -I -m unittest discover -s scripts/tests -p 'test_native_h264_cli.py' -v
cargo test --locked --release -p coreav-cli --features native-h264-encoder,native-h264 --lib ffmpeg -- --nocapture
cargo test --locked --release -p coreav-cli --features native-h264-encoder,native-h264 --test docs_site --test strict_safe_limit --test ffprobe_limits --test native_h264_cli
cargo test --locked --release -p coreav-transcode --features h264-native --lib
cargo clippy --locked -p coreav-cli --features native-h264-encoder,native-h264 --all-targets -- -D warnings
cargo build --locked --release -p coreav-cli --features native-h264-encoder,native-h264
cargo build --locked --release -p coreav-h264-encoder --features full --example p
- name: Install pinned black-box tools and generate trusted fixtures
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends nasm
echo 'Installing pinned FFmpeg'
scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/cli-oracles" || { echo '::error::reason=pinned_ffmpeg_install_failed'; exit 1; }
echo 'Installing pinned x264'
scripts/install-x264.sh "$RUNNER_TEMP/cli-oracles" || { echo '::error::reason=pinned_x264_install_failed'; exit 1; }
export PATH="$RUNNER_TEMP/cli-oracles:$PATH"
scripts/gen-fixtures.sh "$RUNNER_TEMP/cli-fixtures"
- name: Verify stored native output and unchanged external behavior
run: |
python3 -I scripts/native_h264_cli.py \
--candidate target/release/coreav \
--default-candidate "$RUNNER_TEMP/default-candidate" \
--baseline-default "$RUNNER_TEMP/default-base" \
--baseline-external "$RUNNER_TEMP/external-base" \
--driver target/release/examples/p \
--ffmpeg "$RUNNER_TEMP/cli-oracles/ffmpeg" \
--output "$RUNNER_TEMP/native-cli"
- name: Retain encoder timeout and staging regressions
env:
COREAV_FFMPEG: ${{ runner.temp }}/cli-oracles/ffmpeg
COREAV_X264: ${{ runner.temp }}/cli-oracles/x264
COREAV_FIXTURES: ${{ runner.temp }}/cli-fixtures
COREAV_REQUIRE_FFMPEG: '1'
run: cargo test --locked --release -p coreav-cli --features native-h264-encoder,native-h264 --test transcode hostile_encoders_end_in_an_error_and_leave_nothing -- --exact
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: native-h264-cli
path: ${{ runner.temp }}/native-cli/summary.json
if-no-files-found: ignore
h264-encoder-conformance:
runs-on: ubuntu-24.04
timeout-minutes: 30
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/coreav-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ ffmpeg-fallback = ["thumbnail", "transcode", "dep:coreav-worker", "coreav-worker
# in this process. Opt-in and never in a default build: decoding H.264 may
# need a patent licence where you are (docs/design/h264.md, "Packaging").
native-h264 = ["thumbnail", "transcode", "dep:coreav-backend-h264"]
native-h264-encoder = ["transcode", "coreav-transcode/h264-native"]
# Experimental native profile-0 VP9, not in a default or release build.
native-vp9 = ["thumbnail", "transcode", "dep:coreav-backend-vp9"]
native-hevc = ["thumbnail", "transcode", "dep:coreav-backend-hevc", "coreav-backend-hevc/native-hevc"]
Expand Down
67 changes: 58 additions & 9 deletions crates/coreav-cli/src/compat_cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,24 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
Ok(args) => args,
Err(code) => return code,
};
match ffmpeg::args::summary(&args) {
Ok(Some(options)) => return run_summary(Tool::Ffmpeg, &args, &options, strict_safe),
run_ffmpeg_with(
&args,
strict_safe,
#[cfg(feature = "native-h264-encoder")]
None,
)
}

fn run_ffmpeg_with(
args: &[String],
strict_safe: StrictSafe,
#[cfg(feature = "native-h264-encoder")] attempt: Option<ffmpeg::native_h264::EncodeAttempt>,
) -> ExitCode {
match ffmpeg::args::summary(args) {
Ok(Some(options)) => return run_summary(Tool::Ffmpeg, args, &options, strict_safe),
Ok(None) => {}
Err(Refusal::NotNative(refusal)) => {
return fallback_cmd::refuse(Tool::Ffmpeg, &args, refusal, strict_safe);
return fallback_cmd::refuse(Tool::Ffmpeg, args, refusal, strict_safe);
}
Err(Refusal::Usage(text)) => {
message("ffmpeg", &text);
Expand All @@ -87,7 +100,7 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
Err(Refusal::Help) => return print_usage(ffmpeg::USAGE),
}
let mut spool = None;
if let Ok(command) = ffmpeg::args::parse(&args)
if let Ok(command) = ffmpeg::args::parse(args)
&& ffmpeg::args::stdin(&command.input)
&& command.more_inputs.is_empty()
&& matches!(
Expand All @@ -108,17 +121,32 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
}
}
let result = match &spool {
Some(input) => ffmpeg::run_spooled(&args, strict_safe, &input.path),
None => ffmpeg::run(&args, strict_safe),
Some(input) => ffmpeg::run_spooled(args, strict_safe, &input.path),
#[cfg(feature = "native-h264-encoder")]
None => match attempt {
Some(attempt) => ffmpeg::run_attempt(args, strict_safe, attempt),
None => ffmpeg::run(args, strict_safe),
},
#[cfg(not(feature = "native-h264-encoder"))]
None => ffmpeg::run(args, strict_safe),
};
finish_ffmpeg(args, strict_safe, spool, result)
}

fn finish_ffmpeg(
args: &[String],
strict_safe: StrictSafe,
mut spool: Option<EncodedInput>,
result: Result<serde_json::Value, Stop>,
) -> ExitCode {
match result {
Ok(_) => ExitCode::SUCCESS,
Err(Stop::Refused(Refusal::NotNative(refusal))) => {
if let Some(input) = spool.take() {
match input.into_replay() {
Ok((file, input)) => fallback_cmd::refuse_with_input(
Tool::Ffmpeg,
&args,
args,
refusal,
strict_safe,
file,
Expand All @@ -133,7 +161,7 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
}
}
} else {
fallback_cmd::refuse(Tool::Ffmpeg, &args, refusal, strict_safe)
fallback_cmd::refuse(Tool::Ffmpeg, args, refusal, strict_safe)
}
}
Err(Stop::Refused(Refusal::Usage(text))) => {
Expand All @@ -155,7 +183,7 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
.get("message")
.and_then(serde_json::Value::as_str)
.map_or_else(|| json.to_string(), str::to_owned);
message("ffmpeg", &format!("{}: {text}", args_input(&args)));
message("ffmpeg", &format!("{}: {text}", args_input(args)));
ExitCode::from(if failure.is_io() {
EXIT_IO
} else {
Expand All @@ -165,6 +193,27 @@ pub fn run_ffmpeg(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
}
}

#[cfg(feature = "native-h264-encoder")]
pub fn run_ffmpeg_native(args: &[OsString], strict_safe: StrictSafe) -> ExitCode {
let text = match strings("ffmpeg", args) {
Ok(text) => text,
Err(code) => return code,
};
let deadline = std::time::Instant::now() + ffmpeg::ENCODE_LIMIT;
match ffmpeg::native_h264::run(&text, strict_safe, deadline) {
Ok(Some(_)) => ExitCode::SUCCESS,
Ok(None) | Err(Stop::Refused(Refusal::NotNative(_))) => run_ffmpeg_with(
&text,
strict_safe,
Some(ffmpeg::native_h264::EncodeAttempt {
native: None,
deadline,
}),
),
Err(error) => finish_ffmpeg(&text, strict_safe, None, Err(error)),
}
}

pub(super) struct EncodedInput {
path: std::path::PathBuf,
file: Option<std::fs::File>,
Expand Down
32 changes: 30 additions & 2 deletions crates/coreav-cli/src/ffmpeg/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,10 @@ struct Grouped<'a> {
}

fn grouped(args: &[String]) -> Result<Grouped<'_>, Refusal> {
grouped_with_qp(args, false)
}

fn grouped_with_qp(args: &[String], native_qp: bool) -> Result<Grouped<'_>, Refusal> {
let mut inputs: Vec<(&str, Group)> = Vec::new();
let mut outputs: Vec<(&str, Group)> = Vec::new();
let (mut yes, mut no) = (false, false);
Expand Down Expand Up @@ -711,7 +715,10 @@ fn grouped(args: &[String]) -> Result<Grouped<'_>, Refusal> {
.chain(PER_FILE.iter())
.find(|(name, _)| *name == arg)
.map(|(_, arity)| *arity)
.or_else(|| (arg == "-i" || codec_option(arg).is_some()).then_some(Arity::Value));
.or_else(|| {
(arg == "-i" || codec_option(arg).is_some() || native_qp && arg == "-qp")
.then_some(Arity::Value)
});
let Some(arity) = arity else {
let base = arg.split(':').next().unwrap_or(arg);
let why = REFUSED.iter().find(|(name, _)| *name == base).map_or(
Expand Down Expand Up @@ -807,13 +814,34 @@ pub fn summary(args: &[String]) -> Result<Option<Vec<crate::ffprobe::args::Optio

/// Parses an invocation that writes an output file.
pub fn parse(args: &[String]) -> Result<Command, Refusal> {
parse_grouped(grouped(args)?)
}

#[cfg(feature = "native-h264-encoder")]
pub(super) fn native_h264(args: &[String]) -> Result<(Command, Option<String>), Refusal> {
let mut grouped = grouped_with_qp(args, true)?;
let mut qp = None;
if let [(_, options)] = grouped.outputs.as_mut_slice() {
options.retain(|(option, value)| {
if *option == "-qp" {
qp = value.map(str::to_owned);
false
} else {
true
}
});
}
Ok((parse_grouped(grouped)?, qp))
}

fn parse_grouped(grouped: Grouped<'_>) -> Result<Command, Refusal> {
let Grouped {
inputs,
outputs,
yes,
no,
..
} = grouped(args)?;
} = grouped;
let overwrite = match (yes, no) {
(true, true) => return Err(usage("Error, both -y and -n supplied. Exiting.")),
(true, false) => Overwrite::Yes,
Expand Down
21 changes: 20 additions & 1 deletion crates/coreav-cli/src/ffmpeg/encode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,10 @@ pub fn request(
}
};
Ok(Request {
#[cfg(feature = "native-h264-encoder")]
native_h264: None,
#[cfg(feature = "native-h264-encoder")]
deadline: None,
matroska_output: command.muxer.matroska(),
crf,
bitrate,
Expand Down Expand Up @@ -573,7 +577,15 @@ pub fn write(
encoder: Encoder,
strict_safe: StrictSafe,
) -> Result<serde_json::Value, Stop> {
write_inner(command, job, encoder, strict_safe, None)
write_inner(
command,
job,
encoder,
strict_safe,
None,
#[cfg(feature = "native-h264-encoder")]
None,
)
}

pub(super) fn write_inner(
Expand All @@ -582,8 +594,15 @@ pub(super) fn write_inner(
encoder: Encoder,
strict_safe: StrictSafe,
capture: Option<&Path>,
#[cfg(feature = "native-h264-encoder")] attempt: Option<super::native_h264::EncodeAttempt>,
) -> Result<serde_json::Value, Stop> {
let request = request(command, job, encoder, strict_safe)?;
#[cfg(feature = "native-h264-encoder")]
let request = Request {
native_h264: attempt.and_then(|attempt| attempt.native),
deadline: attempt.map(|attempt| attempt.deadline),
..request
};
#[cfg(feature = "input-capture")]
let request = {
let mut request = request;
Expand Down
Loading
Loading