diff --git a/quest/m1/README.md b/quest/m1/README.md index f8463844b1e..cccf47caf2b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -55,7 +55,10 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [OBS native codecs](/quest/m1/obs-moq-video/README.md) - remove FFmpeg decoding dependencies, deliver GPU frames, and use native audio/video encoders - [Opus concealment](/quest/m1/opus-conceal.md) - a lost Opus packet conceals the last packet's length, not 120 ms - [Audio codecs](/quest/m1/audio-codecs/README.md) - platform audio codecs, explicit unsupported cases, and channel layouts up to 7.1 -- [CMAF Opus](/quest/m1/cmaf-opus-dops.md) - fMP4 import and export keep the Opus pre-skip and gain +- [Opus catalog rate](/quest/m1/opus-catalog-rate.md) - MKV Opus import publishes the 48 kHz codec rate in the catalog, not the OpusHead input rate +- [mp4-atom dOps mapping](/quest/m1/mp4-atom-dops-mapping.md) - a released mp4-atom reads and writes any `dOps` channel mapping family and table +- [CMAF surround Opus](/quest/m1/cmaf-opus-surround.md) - fMP4 import and export carry an Opus channel mapping table +- [js/hang dOps pre-skip](/quest/m1/js-dops-pre-skip.md) - CMAF encoding in js/hang stops hard-coding a 312-sample pre-skip - [GPU pool reservation](/quest/m1/gpu-pool-reservation.md) - a full GPU frame pool is a `None` reservation the caller drops on, not an error to match - [Egress rendition pick](/quest/m1/egress-rendition-pick.md) - WHEP and single-track RTMP/FLV serve the best rendition, not the first by name - [Keyframe trigger](/quest/m1/keyframe-trigger.md) - an application can ask the built-in capture encoder for a keyframe diff --git a/quest/m1/cmaf-opus-dops.md b/quest/m1/cmaf-opus-dops.md deleted file mode 100644 index 0456fb6f253..00000000000 --- a/quest/m1/cmaf-opus-dops.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Carry Opus pre-skip and gain through CMAF - -## Goal - -An Opus track imported from or exported to fMP4 keeps the pre-skip and output -gain its `dOps` box declares, so it decodes the same as the OpusHead it came -from. - -## Plan - -The fMP4 importer builds an Opus catalog entry from the sample entry alone and -publishes no description, so a CMAF Opus track decodes with no pre-skip or -gain. Build the OpusHead from `dOps` (input rate, channels, pre-skip, gain) -with `moq_mux::codec::opus::Config` and publish it as the description. The -exporter writes `dOps.output_gain` as 0; take it from the parsed head like the -pre-skip. Refuse a `dOps` channel mapping family other than 0 if `mp4-atom` -exposes one, rather than dropping its table. - -Regression: an fMP4 Opus fixture with nonzero pre-skip and gain round-trips -through import and export with both preserved. diff --git a/quest/m1/cmaf-opus-surround.md b/quest/m1/cmaf-opus-surround.md new file mode 100644 index 00000000000..da3ae24b669 --- /dev/null +++ b/quest/m1/cmaf-opus-surround.md @@ -0,0 +1,22 @@ +# [S] Carry surround Opus through CMAF + +## Goal + +An fMP4 Opus track whose `dOps` declares channel mapping family 1 imports with +its mapping table in the OpusHead description, and a family 1 description +exports to a `dOps` that keeps the table. Today both directions refuse it. + +## Plan + +Build the head from the `dOps` family and table with `opus::Mapping` on import, +and write the table from the parsed head on export in place of the +`UnsupportedMappingFamily` refusal in `synthesize_audio_trak`. Keep refusing +families the head cannot describe. + +Regression: a family 1 5.1 `dOps` round-trips through import and export with +its table intact. + +## Required + +- [mp4-atom dOps mapping](/quest/m1/mp4-atom-dops-mapping.md) - `Dops` carries the mapping family and table +- [Audio codecs](/quest/m1/audio-codecs/README.md) - `opus::Mapping` and a `Config::encode` that writes any family's table diff --git a/quest/m1/js-dops-pre-skip.md b/quest/m1/js-dops-pre-skip.md new file mode 100644 index 00000000000..273669a0f91 --- /dev/null +++ b/quest/m1/js-dops-pre-skip.md @@ -0,0 +1,12 @@ +# [XS] js/hang dOps without an invented pre-skip + +## Goal + +`js/hang` CMAF encoding stops writing a hard-coded 312-sample pre-skip into +`dOps` when the track has no description, so a remuxed track never trims audio +its encoder did not ask to trim. + +## Plan + +Without an OpusHead there is no known pre-skip; write 0 or refuse, whichever +matches how the Rust exporter (`synthesize_audio_trak`) handles the same case. diff --git a/quest/m1/mp4-atom-dops-mapping.md b/quest/m1/mp4-atom-dops-mapping.md new file mode 100644 index 00000000000..3d1f4d818de --- /dev/null +++ b/quest/m1/mp4-atom-dops-mapping.md @@ -0,0 +1,15 @@ +# [S] mp4-atom dOps channel mapping + +## Goal + +A released `mp4-atom` decodes and encodes a `dOps` box with any channel mapping +family, exposing the family and its table (stream count, coupled count, and the +per-channel mapping) instead of refusing a nonzero family. This repository +bumps to that release. + +## Plan + +The work lands in kixelated/mp4-atom. Validate the table against the output +channel count on decode rather than trusting it, and keep family 0 encoding +byte-identical. The bump here is a separate, small step once the release +ships. diff --git a/quest/m1/opus-catalog-rate.md b/quest/m1/opus-catalog-rate.md new file mode 100644 index 00000000000..d6ba6bdd842 --- /dev/null +++ b/quest/m1/opus-catalog-rate.md @@ -0,0 +1,15 @@ +# [XS] Opus catalog rate matches the decoder + +## Goal + +MKV Opus import publishes the codec rate (48 kHz) as the catalog sample rate, +not the OpusHead input rate, which is informational. The catalog describes +what the decoder outputs. + +## Plan + +fMP4 import already does this. Check the other importers that build the +catalog from an OpusHead (FLV and the raw Opus import look similar) and align +them in the same PR. The OpusHead description keeps the input rate. + +Regression: a 44.1 kHz-input OpusHead imports with a 48 kHz catalog rate. diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index c5d20ebac6f..1f454c3fea7 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -873,6 +873,25 @@ fn synthesize_opus_trak_preserves_pre_skip() { assert_eq!(opus.dops.pre_skip, 312); } +/// dOps has nowhere to put a channel mapping table, so a family 1 head is refused rather +/// than written as family 0 without it. +#[test] +fn synthesize_opus_trak_refuses_mapping_family() { + use hang::catalog::{AudioCodec, AudioConfig}; + + let mut head = crate::codec::opus::Config::new(48_000, 2).encode().unwrap().to_vec(); + head[18] = 1; // channel mapping family + head.extend_from_slice(&[1, 1, 0, 1]); // one coupled stream feeding both channels + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + config.description = Some(head.into()); + + let err = super::synthesize_audio_trak(1, 48_000, &config).unwrap_err(); + assert!(matches!( + err, + super::Error::Opus(crate::codec::opus::Error::UnsupportedMappingFamily(1)) + )); +} + /// A legacy FLAC rendition (no init segment) synthesizes a `fLaC` sample entry /// whose `dfLa` STREAMINFO is rebuilt from the catalog description. #[test] diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 15d2cf26eac..d30facd4e4f 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -561,11 +561,19 @@ impl Import { config } mp4_atom::Codec::Opus(opus) => { - let mut config = AudioConfig::new( - AudioCodec::Opus, - opus.audio.sample_rate.integer() as _, - opus.audio.channel_count as _, - ); + // dOps carries the OpusHead fields in ISOBMFF byte order; republish them as the + // OpusHead description so decoders apply its pre-skip and gain. mp4-atom already + // refuses a nonzero channel mapping family, so the table is never dropped here. + let dops = &opus.dops; + let mut head = + crate::codec::opus::Config::new(dops.input_sample_rate, dops.output_channel_count as u32) + .with_pre_skip(dops.pre_skip); + head.output_gain = dops.output_gain; + + // The catalog describes the decoder's output: Opus always decodes at 48 kHz, whatever + // the sample entry or the informational input rate claims. + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, head.channel_count); + config.description = Some(head.encode()?); config.container = container; config } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 9d54992e802..40c10a626d6 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -554,7 +554,24 @@ fn test_flac_catalog() { }, }; - let trak = super::build_audio_trak(1, 96_000, mp4_atom::Codec::from(flac)); + let data = audio_init(super::build_audio_trak(1, 96_000, mp4_atom::Codec::from(flac))); + + let catalog = run_fmp4(&data); + assert_eq!(catalog.audio.renditions.len(), 1); + + let a = catalog.audio.renditions.values().next().unwrap(); + assert!(matches!(a.codec, hang::catalog::AudioCodec::Flac)); + assert_eq!(a.sample_rate, 96_000); + assert_eq!(a.channel_count, 2); + // fmp4 import is CMAF passthrough. + assert!(matches!(a.container, Container::Cmaf { .. })); + // The WebCodecs FLAC description: `fLaC` marker + STREAMINFO. + let desc = a.description.as_ref().expect("flac description"); + assert_eq!(&desc[..4], b"fLaC"); +} + +/// An init segment (ftyp + moov) holding a single audio trak with track ID 1. +fn audio_init(trak: mp4_atom::Trak) -> Vec { let moov = mp4_atom::Moov { mvhd: mp4_atom::Mvhd { timescale: 1000, @@ -580,19 +597,83 @@ fn test_flac_catalog() { let mut data = Vec::new(); ftyp.encode(&mut data).unwrap(); moov.encode(&mut data).unwrap(); + data +} - let catalog = run_fmp4(&data); - assert_eq!(catalog.audio.renditions.len(), 1); +/// An Opus init segment whose dOps declares a 44.1 kHz input, 312 samples of pre-skip, +/// and -6 dB of gain, behind a sample entry claiming `entry_rate`. +fn opus_init(entry_rate: u16) -> (mp4_atom::Dops, Vec) { + let dops = mp4_atom::Dops { + output_channel_count: 2, + pre_skip: 312, + input_sample_rate: 44_100, + output_gain: -1536, + }; + let opus = mp4_atom::Opus { + audio: mp4_atom::Audio { + data_reference_index: 1, + channel_count: 2, + sample_size: 16, + sample_rate: mp4_atom::FixedPoint::from(entry_rate), + }, + dops: dops.clone(), + btrt: None, + }; + let data = audio_init(super::build_audio_trak(1, 48_000, mp4_atom::Codec::from(opus))); + (dops, data) +} - let a = catalog.audio.renditions.values().next().unwrap(); - assert!(matches!(a.codec, hang::catalog::AudioCodec::Flac)); - assert_eq!(a.sample_rate, 96_000); +/// dOps becomes the OpusHead description, and a track re-exported from that description +/// writes the same dOps back, so pre-skip and gain survive the round trip. +#[test] +fn opus_dops_round_trips() { + let (dops, data) = opus_init(48_000); + + let catalog = run_fmp4(&data); + let a = catalog.audio.renditions.values().next().expect("opus rendition"); + assert!(matches!(a.codec, hang::catalog::AudioCodec::Opus)); + assert_eq!(a.sample_rate, 48_000); assert_eq!(a.channel_count, 2); - // fmp4 import is CMAF passthrough. - assert!(matches!(a.container, Container::Cmaf { .. })); - // The WebCodecs FLAC description: `fLaC` marker + STREAMINFO. - let desc = a.description.as_ref().expect("flac description"); - assert_eq!(&desc[..4], b"fLaC"); + + let desc = a.description.as_ref().expect("opus description"); + let head = crate::codec::opus::Config::parse(&mut desc.as_ref()).unwrap(); + assert_eq!(head.sample_rate, 44_100); + assert_eq!(head.channel_count, 2); + assert_eq!(head.pre_skip, 312); + assert_eq!(head.output_gain, -1536); + + let trak = super::synthesize_audio_trak(1, 48_000, a).expect("synthesize Opus trak"); + match &trak.mdia.minf.stbl.stsd.codecs[0] { + mp4_atom::Codec::Opus(opus) => assert_eq!(opus.dops, dops), + other => panic!("expected Opus sample entry, got {other:?}"), + } +} + +/// The catalog describes the decoder's 48 kHz output even when the sample entry claims +/// another rate. +#[test] +fn opus_catalog_uses_the_decode_rate() { + let (_, data) = opus_init(44_100); + let catalog = run_fmp4(&data); + let a = catalog.audio.renditions.values().next().expect("opus rendition"); + assert_eq!(a.sample_rate, 48_000); +} + +/// A dOps with a channel mapping table is refused rather than imported without it. +#[test] +fn opus_dops_mapping_family_is_refused() { + let (_, mut data) = opus_init(48_000); + + // The mapping family byte follows version, channels, pre-skip, rate, and gain. + let at = data.windows(4).position(|w| w == b"dOps").unwrap() + 4 + 10; + assert_eq!(data[at], 0); + data[at] = 1; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + assert!(fmp4.decode(&bytes::BytesMut::from(data.as_slice())).is_err()); + assert!(catalog.snapshot().audio.renditions.is_empty()); } // ---- Segment-driven grouping ---- diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 30af4a65c10..f44a39e698a 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -715,20 +715,24 @@ pub(crate) fn synthesize_audio_trak(track_id: u32, timescale: u64, config: &Audi let sample_entry = match &config.codec { AudioCodec::Opus => { - let pre_skip = match &config.description { + let head = match &config.description { Some(description) => { - let mut description = description.as_ref(); - crate::codec::opus::Config::parse(&mut description)?.pre_skip + let head = crate::codec::opus::Config::parse(&mut description.as_ref())?; + // dOps shares OpusHead's family 0 layout; a mapping table would need writing too. + if head.mapping_family != 0 { + return Err(crate::codec::opus::Error::UnsupportedMappingFamily(head.mapping_family).into()); + } + head } - None => 0, + None => crate::codec::opus::Config::new(config.sample_rate, config.channel_count), }; mp4_atom::Codec::from(mp4_atom::Opus { audio, dops: mp4_atom::Dops { output_channel_count: config.channel_count as u8, - pre_skip, - input_sample_rate: config.sample_rate, - output_gain: 0, + pre_skip: head.pre_skip, + input_sample_rate: head.sample_rate, + output_gain: head.output_gain, }, btrt: None, })