From 289c2a3a3584b4d14d5e7798621fc5a38d3c05a6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:42:59 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/premultiply.md Quest-Claim: 2a06fa636f663bc1 From 30cfdb9ab4ec1310e84d31e680da98dd0eaa5dbd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:33:59 -0800 Subject: [PATCH 2/5] fix(scene): sample transparent pictures premultiplied Bilinear sampling mixed straight-alpha texels, so a scaled picture took on the color of its transparent neighbours at its edges. Stills now upload premultiplied and canvases and rendered frames, being opaque, are marked so, keeping them on the sampler's fast path. An overlay's straight paint, and a straight mask, premultiply each of their four taps in the shader before mixing them. Co-Authored-By: Claude Opus 5.5 --- quest/m0/README.md | 1 - quest/m0/premultiply.md | 23 ------- rs/moq-scene/src/compositor.rs | 115 +++++++++++++++++++++++++++++++-- rs/moq-scene/src/effect.rs | 2 +- rs/moq-scene/src/effect.wgsl | 43 +++++++++--- 5 files changed, 144 insertions(+), 40 deletions(-) delete mode 100644 quest/m0/premultiply.md diff --git a/quest/m0/README.md b/quest/m0/README.md index eb29ff61..3fb1ceac 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -106,7 +106,6 @@ repository. - [Virtual camera](/quest/m0/vcam/README.md) - a canvas's program exposed as a system camera on Linux, Windows, and macOS - [Clean quit](/quest/m0/clean-quit.md) - quitting finishes every track and closes the relay session, so the broadcast's tail arrives - [Recover crashed recordings](/quest/m0/recording-repair.md) - a recording a crash left unfinished is repaired beside the original at launch -- [Premultiplied source sampling](/quest/m0/premultiply.md) - scaled transparent sources draw without edge fringes - [Deterministic browser source startup](/quest/m0/cef-startup.md) - CEF on the CPU path paints its first frame without the GPU fallback hang - [A browser renderer that dies on its first page](/quest/m0/browser-first-crash.md) - a renderer lost during the first navigation fails its source - [Signing identities are available](/quest/m0/signing.md) - the Apple and Windows code-signing credentials packaging and the macOS camera need diff --git a/quest/m0/premultiply.md b/quest/m0/premultiply.md deleted file mode 100644 index f8bd9013..00000000 --- a/quest/m0/premultiply.md +++ /dev/null @@ -1,23 +0,0 @@ -# [XS] Premultiplied source sampling - -## Goal - -A scaled source with transparency, such as a PNG logo, draws without a dark -or light fringe at its edges, because the compositor samples it -premultiplied. - -## Plan - -From [moq-dev/studio#17](https://github.com/moq-dev/studio/pull/17), raised -by CodeRabbit on `rs/moq-scene/src/effect.wgsl`. It predates that PR. - -- Source pictures are sampled bilinearly straight (not premultiplied), so a - transparent texel's color bleeds into its neighbours when scaled. - Premultiply before interpolation: upload premultiplied, or premultiply each - tap in the shader, whichever adds no GPU copy. The effect pass's - `premultiplied` flag is not the fix: `fetch()` in `effect.wgsl` interpolates - first and `straight()` divides afterwards, so the flag only describes the - input. -- A compositor test scales a transparent picture with a colored transparent - region and checks that edge texels carry no color from it. -- Measure frame time before and after. diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index 492c612b..1365e4f2 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -352,8 +352,10 @@ impl Compositor { overlay: None, error: None, }); - // A page arrives premultiplied; frames and images are straight. - let premultiplied = matches!(picture, source::Picture::Texture(_)); + // Every picture lands premultiplied, so its texels mix on the + // sampler's fast path: a page arrives so, moq-video renders a + // frame opaque, and a still is premultiplied here. Only an + // overlay's paint is straight, for the shader to premultiply. let texture = match picture { source::Picture::Frame(frame) => { let renderer = match layer.renderer.as_mut() { @@ -365,7 +367,10 @@ impl Compositor { .and_then(|renderer| renderer.render(&frame)) .map_err(Error::from) } - source::Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + source::Picture::Rgba { size, mut pixels } => { + premultiply(&mut pixels); + upload(&self.device, &self.queue, size, &pixels) + } source::Picture::Texture(texture) => Ok(texture), source::Picture::Lut(cube) => { layer.lut = Some(self.effects.lut(&cube)); @@ -394,7 +399,7 @@ impl Compositor { match texture { Ok(texture) => { layer.error = None; - layer.picture = Some(self.effects.picture(&texture, premultiplied)); + layer.picture = Some(self.effects.picture(&texture, true)); } Err(err) => { if layer.error.is_none() { @@ -772,6 +777,16 @@ impl Draw<'_> { } } +/// Premultiply tightly packed RGBA in place. +fn premultiply(pixels: &mut [u8]) { + for pixel in pixels.chunks_exact_mut(4) { + let alpha = u16::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u16::from(*channel) * alpha + 127) / 255) as u8; + } + } +} + /// A texture holding tightly packed RGBA pixels. fn upload( device: &wgpu::Device, @@ -838,7 +853,8 @@ impl Target { | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }); - let picture = effects.picture(&texture, false); + // A canvas is opaque, which is premultiplied too. + let picture = effects.picture(&texture, true); Self { texture, picture } } } @@ -1760,6 +1776,95 @@ mod tests { .await; } + /// A scaled picture mixes its texels premultiplied, so the color of a + /// transparent texel bleeds into no edge, of a still, an overlay's + /// straight paint, or a luminance mask, while the edge still fades. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn scaling_bleeds_no_transparent_color() { + local(async { + let dir = tempfile::tempdir().unwrap(); + // Red beside a transparent green, which straight mixing tints the + // edge with. + image::RgbaImage::from_fn(2, 1, |x, _| image::Rgba(if x == 0 { RED } else { [0, 255, 0, 0] })) + .save(dir.path().join("logo.png")) + .unwrap(); + // White, black, then a transparent white, which straight mixing + // lets show through where the mask is black. + image::RgbaImage::from_fn(3, 1, |x, _| { + image::Rgba([[255; 4], BLACK, [255, 255, 255, 0]][x as usize]) + }) + .save(dir.path().join("mask.png")) + .unwrap(); + let mut collection = collection(); + for name in ["logo", "mask"] { + collection.sources.insert( + name.into(), + Source::Image { + path: format!("{name}.png").into(), + }, + ); + } + // White beside transparent, painted straight. + collection.sources.insert( + "overlay".into(), + Source::Overlay { + html: "
" + .into(), + width: 2, + height: 1, + }, + ); + let row = |source: &str, y| { + Item::new( + source, + Rect { + x: 0.0, + y, + width: 8.0, + height: 2.0, + }, + ) + }; + collection.scenes[0].items = vec![ + row("logo", 0.0), + row("overlay", 2.0), + Item { + filters: vec![Filter::Mask { + source: "mask".into(), + mode: MaskMode::Luminance, + }], + ..row("blue", 4.0) + }, + ]; + let program = drawn(collection, dir).await; + + for x in 0..8 { + let logo = pixel(&program, x, 0); + assert_eq!(logo[1..3], [0, 0], "logo at {x}: {logo:?}"); + // The overlay's edge fades as the logo's does. + let overlay = pixel(&program, x, 2); + assert!( + near(overlay, rgb([logo[0]; 3])), + "overlay at {x}: {overlay:?}, logo {logo:?}" + ); + } + assert_eq!(pixel(&program, 0, 0), RED); + let edge = pixel(&program, 3, 0)[0]; + assert!(0 < edge && edge < 255, "the logo's edge fades: {edge}"); + assert_eq!(pixel(&program, 7, 0), BLACK); + + assert_eq!(pixel(&program, 0, 4), BLUE); + for x in 4..8 { + assert_eq!(pixel(&program, x, 4), BLACK, "mask at {x}"); + } + }) + .await; + } + /// A resize draws the next frame on textures of the new size, with items /// where their pixels say, a blend mode and a blur reaching the canvas's /// new width, and a canvas the GPU cannot hold changes nothing. diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs index b1a0ca12..4a112f44 100644 --- a/rs/moq-scene/src/effect.rs +++ b/rs/moq-scene/src/effect.rs @@ -949,7 +949,7 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: false, + premultiplied: picture.premultiplied, }, segment: Segment { ops: ops.into_iter().collect(), diff --git a/rs/moq-scene/src/effect.wgsl b/rs/moq-scene/src/effect.wgsl index 40d503a6..49cc2a25 100644 --- a/rs/moq-scene/src/effect.wgsl +++ b/rs/moq-scene/src/effect.wgsl @@ -1,7 +1,8 @@ // Every draw the compositor makes: a quad sampling one picture, through a list // of per-pixel filters, a blur, or a drop shadow. Colors are gamma-encoded -// sRGB, as CSS filters see them. Rasters between passes hold premultiplied -// alpha, so a blur weighs a transparent pixel as nothing. +// sRGB, as CSS filters see them. Every picture is sampled premultiplied, and +// rasters between passes hold premultiplied alpha, so neither scaling nor a +// blur mixes in a transparent pixel's color. const OPS: u32 = 8u; @@ -79,20 +80,43 @@ fn within(uv: vec2, low: vec2, high: vec2) -> bool { return all(uv >= low) && all(uv <= high); } -// The input at `uv`, transparent outside its picture, as CSS's edge mode none. +// One texel of a straight-alpha texture, premultiplied. +fn texel(texture: texture_2d, at: vec2) -> vec4 { + let color = textureLoad(texture, at, 0); + return vec4(color.rgb * color.a, color.a); +} + +// `texture` at `uv`, mixed bilinearly with premultiplied alpha. A straight +// texture's four nearest texels are premultiplied before they mix, so a +// transparent texel's color, which shows nowhere, bleeds into no neighbour. +fn filtered(texture: texture_2d, uv: vec2, premultiplied: bool) -> vec4 { + if premultiplied { + return textureSampleLevel(texture, bilinear, uv, 0.0); + } + // The sampler's clamp to the edge, by hand. + let size = vec2(textureDimensions(texture)); + let at = uv * vec2(size) - 0.5; + let low = vec2(floor(at)); + let near = clamp(low, vec2(0), size - 1); + let far = clamp(low + 1, vec2(0), size - 1); + let fraction = at - floor(at); + let top = mix(texel(texture, near), texel(texture, vec2(far.x, near.y)), fraction.x); + let bottom = mix(texel(texture, vec2(near.x, far.y)), texel(texture, far), fraction.x); + return mix(top, bottom, fraction.y); +} + +// The input at `uv`, premultiplied, and transparent outside its picture, as +// CSS's edge mode none. fn fetch(uv: vec2) -> vec4 { if !within(uv, draw.valid.xy, draw.valid.zw) { return vec4(0.0); } - return textureSampleLevel(input, bilinear, uv, 0.0); + return filtered(input, uv, draw.mode.y != 0u); } // The input at `uv`, with straight alpha. fn straight(uv: vec2) -> vec4 { let color = fetch(uv); - if draw.mode.y == 0u { - return color; - } if color.a <= 0.0 { return vec4(0.0); } @@ -142,10 +166,9 @@ fn coverage(item: vec2, op: Op) -> f32 { if !within(item, vec2(0.0), vec2(1.0)) { return 0.0; } - let color = textureSampleLevel(mask, bilinear, item, 0.0); + let color = filtered(mask, item, op.params.y > 0.5); if op.params.x > 0.5 { - let rgb = select(color.rgb * color.a, color.rgb, op.params.y > 0.5); - return dot(rgb, LUMA); + return dot(color.rgb, LUMA); } return color.a; } From f28cd26c537ac1125c083078fd194f459e50f0a9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:21:26 -0800 Subject: [PATCH 3/5] perf(scene): sample a straight picture's texel centers once A picture drawn at its own size samples on texel centers, where one texel is the whole mix, so it skips the four loads. Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/effect.rs | 3 ++- rs/moq-scene/src/effect.wgsl | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs index 4a112f44..bf94a422 100644 --- a/rs/moq-scene/src/effect.rs +++ b/rs/moq-scene/src/effect.rs @@ -323,7 +323,8 @@ fn reach(sigma: f32) -> f32 { pub(crate) struct Picture { view: wgpu::TextureView, bind: wgpu::BindGroup, - /// Whether its alpha is premultiplied, as a web page's is. + /// Whether its alpha is premultiplied, as every picture's is but an + /// overlay's paint, which the shader premultiplies as it samples. premultiplied: bool, } diff --git a/rs/moq-scene/src/effect.wgsl b/rs/moq-scene/src/effect.wgsl index 49cc2a25..b9da7a1a 100644 --- a/rs/moq-scene/src/effect.wgsl +++ b/rs/moq-scene/src/effect.wgsl @@ -93,13 +93,20 @@ fn filtered(texture: texture_2d, uv: vec2, premultiplied: bool) -> vec if premultiplied { return textureSampleLevel(texture, bilinear, uv, 0.0); } - // The sampler's clamp to the edge, by hand. let size = vec2(textureDimensions(texture)); let at = uv * vec2(size) - 0.5; + let fraction = fract(at); + // Near a texel's center, where a picture drawn at its own size samples, + // that texel alone is the mix, to within a 512th of a neighbour. + if all(min(fraction, 1.0 - fraction) < vec2(1.0 / 512.0)) { + let color = textureSampleLevel(texture, bilinear, uv, 0.0); + return vec4(color.rgb * color.a, color.a); + } + // Loaded rather than gathered, which GLES 3.0 lacks, and clamped to the + // edge as the sampler would. let low = vec2(floor(at)); let near = clamp(low, vec2(0), size - 1); let far = clamp(low + 1, vec2(0), size - 1); - let fraction = at - floor(at); let top = mix(texel(texture, near), texel(texture, vec2(far.x, near.y)), fraction.x); let bottom = mix(texel(texture, vec2(near.x, far.y)), texel(texture, far), fraction.x); return mix(top, bottom, fraction.y); From 9984e6073d2bb1c9e68b9ae1e5ea05578a7e3d74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:26:57 -0800 Subject: [PATCH 4/5] fix(scene): premultiply over fixed-size chunks, as clippy asks Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/compositor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index 90461c40..cdb3f95b 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -821,7 +821,7 @@ impl Draw<'_> { /// Premultiply tightly packed RGBA in place. fn premultiply(pixels: &mut [u8]) { - for pixel in pixels.chunks_exact_mut(4) { + for pixel in pixels.as_chunks_mut::<4>().0 { let alpha = u16::from(pixel[3]); for channel in &mut pixel[..3] { *channel = ((u16::from(*channel) * alpha + 127) / 255) as u8; From ceb6137d3f58ec38232879484edd1241b6999166 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 07:56:32 -0800 Subject: [PATCH 5/5] perf(scene): premultiply an overlay once per paint, not per sample Vello paints straight alpha into a texture of its own, and one pass after each paint copies it premultiplied into the texture the compositor samples. Every picture is now premultiplied, so draws and masks take the sampler's hardware path, and the effect pass drops its premultiplied flag and the per-tap bilinear path. Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/compositor.rs | 14 +-- rs/moq-scene/src/effect.rs | 27 ++--- rs/moq-scene/src/effect.wgsl | 48 ++------- rs/moq-scene/src/overlay.rs | 184 ++++++++++++++++++++++++++++----- 4 files changed, 183 insertions(+), 90 deletions(-) diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index 8018b446..e45e0175 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -426,10 +426,10 @@ impl Compositor { overlay: None, error: None, }); - // Every picture lands premultiplied, so its texels mix on the - // sampler's fast path: a page arrives so, moq-video renders a - // frame opaque, and a still is premultiplied here. Only an - // overlay's paint is straight, for the shader to premultiply. + // Every picture lands premultiplied, so scaling it mixes no + // transparent texel's color: a page arrives so, moq-video renders + // a frame opaque, a still is premultiplied here, and an overlay's + // paint as it lands. let texture = match picture { source::Picture::Frame(frame) => { let renderer = match layer.renderer.as_mut() { @@ -461,7 +461,7 @@ impl Compositor { match overlays { Ok(overlays) => { let overlay = overlays.open(html, *width, *height, now); - layer.picture = Some(self.effects.picture(overlay.texture(), false)); + layer.picture = Some(self.effects.picture(overlay.texture())); layer.overlay = Some(overlay); } Err(err) => layer.error = Some(err.to_string()), @@ -473,7 +473,7 @@ impl Compositor { match texture { Ok(texture) => { layer.error = None; - layer.picture = Some(self.effects.picture(&texture, true)); + layer.picture = Some(self.effects.picture(&texture)); } Err(err) => { if layer.error.is_none() { @@ -1008,7 +1008,7 @@ impl Target { view_formats: &[], }); // A canvas is opaque, which is premultiplied too. - let picture = effects.picture(&texture, true); + let picture = effects.picture(&texture); Self { texture, picture } } } diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs index be677bfb..94518ee5 100644 --- a/rs/moq-scene/src/effect.rs +++ b/rs/moq-scene/src/effect.rs @@ -323,9 +323,6 @@ fn reach(sigma: f32) -> f32 { pub(crate) struct Picture { view: wgpu::TextureView, bind: wgpu::BindGroup, - /// Whether its alpha is premultiplied, as every picture's is but an - /// overlay's paint, which the shader premultiplies as it samples. - premultiplied: bool, } /// A LUT source's table on the GPU. @@ -345,7 +342,6 @@ struct Input { area: Area, uv: [f32; 4], valid: [f32; 4], - premultiplied: bool, } /// Per-pixel filters waiting to run when an input is sampled. @@ -663,8 +659,9 @@ impl Effects { } } - /// `texture`, a source's picture or a canvas-sized target, bound to sample. - pub fn picture(&self, texture: &wgpu::Texture, premultiplied: bool) -> Picture { + /// `texture`, a source's picture or a canvas-sized target with + /// premultiplied alpha, bound to sample. + pub fn picture(&self, texture: &wgpu::Texture) -> Picture { let view = texture.create_view(&Default::default()); let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("moq-scene input"), @@ -680,11 +677,7 @@ impl Effects { }, ], }); - Picture { - view, - bind, - premultiplied, - } + Picture { view, bind } } /// Upload a LUT's table. @@ -834,7 +827,6 @@ impl Effects { area, uv: crop, valid: crop, - premultiplied: item.picture.premultiplied, }; let stages: Vec = item.filters.iter().map(|filter| Stage::of(filter)).collect(); @@ -871,7 +863,7 @@ impl Effects { let picture = library.picture(source)?; let op = Op { kind: [Op::MASK, 0, 0, 0], - params: [mode, f32::from(u8::from(picture.premultiplied)), 0.0, 0.0], + params: [mode, 0.0, 0.0, 0.0], ..Default::default() }; (op, None, Some(picture.view)) @@ -952,7 +944,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: picture.premultiplied, }, segment: Segment { ops: ops.into_iter().collect(), @@ -1023,7 +1014,7 @@ impl Effects { input: input.area.map(input.uv, quad), valid: input.valid, item: item.map(UNIT, quad), - mode: [segment.ops.len() as u32, input.premultiplied.into(), 0, 0], + mode: [segment.ops.len() as u32, 0, 0, 0], ops, ..Default::default() } @@ -1049,7 +1040,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: true, } } @@ -1096,7 +1086,6 @@ impl Effects { rect: FULL, input: UNIT, valid: UNIT, - mode: [0, 1, 0, 0], blur, ..Default::default() }; @@ -1133,7 +1122,6 @@ impl Effects { input: picture.area.map(UNIT, &area), valid: UNIT, aux: shape.area.map(UNIT, &area.shift(-offset.0, -offset.1)), - mode: [0, 1, 0, 0], color: color.0.map(|channel| f32::from(channel) / 255.0), ..Default::default() }; @@ -1152,7 +1140,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: true, } } @@ -1190,7 +1177,7 @@ impl Effects { }); Raster { size, - picture: self.picture(&texture, true), + picture: self.picture(&texture), } } }; diff --git a/rs/moq-scene/src/effect.wgsl b/rs/moq-scene/src/effect.wgsl index b9da7a1a..3afcab6a 100644 --- a/rs/moq-scene/src/effect.wgsl +++ b/rs/moq-scene/src/effect.wgsl @@ -1,8 +1,8 @@ // Every draw the compositor makes: a quad sampling one picture, through a list // of per-pixel filters, a blur, or a drop shadow. Colors are gamma-encoded -// sRGB, as CSS filters see them. Every picture is sampled premultiplied, and -// rasters between passes hold premultiplied alpha, so neither scaling nor a -// blur mixes in a transparent pixel's color. +// sRGB, as CSS filters see them. Every picture and every raster between passes +// holds premultiplied alpha, so neither scaling nor a blur mixes in a +// transparent pixel's color. const OPS: u32 = 8u; @@ -20,8 +20,7 @@ const LUMA: vec3 = vec3(0.2126, 0.7152, 0.0722); struct Op { kind: vec4, // MATRIX: unused. CHROMA_KEY: key Cb, key Cr, similarity, smoothness. - // LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode, and 1 if - // the mask is premultiplied. + // LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode. params: vec4, // MATRIX: applied to straight RGBA. LUT: the domain's max in column 0. matrix: mat4x4, @@ -40,7 +39,7 @@ struct Draw { item: vec4, // The aux texture's coordinates at the quad's corners. aux: vec4, - // Ops in x, whether the input is premultiplied in y, the blend mode in z. + // Ops in x, the blend mode in z. mode: vec4, // A blur's step between taps in xy, its standard deviation in z, and its // taps each side in w, all in texels. @@ -80,45 +79,13 @@ fn within(uv: vec2, low: vec2, high: vec2) -> bool { return all(uv >= low) && all(uv <= high); } -// One texel of a straight-alpha texture, premultiplied. -fn texel(texture: texture_2d, at: vec2) -> vec4 { - let color = textureLoad(texture, at, 0); - return vec4(color.rgb * color.a, color.a); -} - -// `texture` at `uv`, mixed bilinearly with premultiplied alpha. A straight -// texture's four nearest texels are premultiplied before they mix, so a -// transparent texel's color, which shows nowhere, bleeds into no neighbour. -fn filtered(texture: texture_2d, uv: vec2, premultiplied: bool) -> vec4 { - if premultiplied { - return textureSampleLevel(texture, bilinear, uv, 0.0); - } - let size = vec2(textureDimensions(texture)); - let at = uv * vec2(size) - 0.5; - let fraction = fract(at); - // Near a texel's center, where a picture drawn at its own size samples, - // that texel alone is the mix, to within a 512th of a neighbour. - if all(min(fraction, 1.0 - fraction) < vec2(1.0 / 512.0)) { - let color = textureSampleLevel(texture, bilinear, uv, 0.0); - return vec4(color.rgb * color.a, color.a); - } - // Loaded rather than gathered, which GLES 3.0 lacks, and clamped to the - // edge as the sampler would. - let low = vec2(floor(at)); - let near = clamp(low, vec2(0), size - 1); - let far = clamp(low + 1, vec2(0), size - 1); - let top = mix(texel(texture, near), texel(texture, vec2(far.x, near.y)), fraction.x); - let bottom = mix(texel(texture, vec2(near.x, far.y)), texel(texture, far), fraction.x); - return mix(top, bottom, fraction.y); -} - // The input at `uv`, premultiplied, and transparent outside its picture, as // CSS's edge mode none. fn fetch(uv: vec2) -> vec4 { if !within(uv, draw.valid.xy, draw.valid.zw) { return vec4(0.0); } - return filtered(input, uv, draw.mode.y != 0u); + return textureSampleLevel(input, bilinear, uv, 0.0); } // The input at `uv`, with straight alpha. @@ -169,11 +136,12 @@ fn lookup(rgb: vec3, op: Op) -> vec3 { } // The mask's coverage at `item`, stretched over the item and empty outside it. +// Luminance is of the premultiplied color, so a transparent texel covers none. fn coverage(item: vec2, op: Op) -> f32 { if !within(item, vec2(0.0), vec2(1.0)) { return 0.0; } - let color = filtered(mask, item, op.params.y > 0.5); + let color = textureSampleLevel(mask, bilinear, item, 0.0); if op.params.x > 0.5 { return dot(color.rgb, LUMA); } diff --git a/rs/moq-scene/src/overlay.rs b/rs/moq-scene/src/overlay.rs index f8e95fda..7d66cebc 100644 --- a/rs/moq-scene/src/overlay.rs +++ b/rs/moq-scene/src/overlay.rs @@ -1,6 +1,6 @@ //! Overlays: titles, lower thirds, and other graphics written as HTML and CSS, -//! laid out by Blitz and painted by Vello straight into a texture on the -//! compositor's device, on the program clock. +//! laid out by Blitz and painted by Vello into a texture on the compositor's +//! device, on the program clock. //! //! An overlay is untrusted, so it loads and runs nothing: Blitz has no script //! engine, its document has no network or navigation provider, and [`check`] @@ -22,6 +22,24 @@ use style::shared_lock::{SharedRwLock, SharedRwLockReadGuard, ToCssWithGuard}; use style::stylesheets::{AllowImportRules, CssRule, CssRuleType, Origin, StylesheetContents, UrlExtraData}; use crate::Error; +use crate::compositor::FORMAT; + +/// Copies an overlay's straight paint into the texture the compositor samples, +/// premultiplied, so scaling it mixes no transparent texel's color. +const PREMULTIPLY: &str = r#" +@group(0) @binding(0) var paint: texture_2d; + +// One triangle covering the whole target. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> @builtin(position) vec4 { + let corner = vec2(f32((index << 1u) & 2u), f32(index & 2u)); + return vec4(corner * 2.0 - 1.0, 0.0, 1.0); +} + +@fragment fn fs_main(@builtin(position) position: vec4) -> @location(0) vec4 { + let color = textureLoad(paint, vec2(position.xy), 0); + return vec4(color.rgb * color.a, color.a); +} +"#; /// The elements an overlay may hold: text and the boxes around it. Anything /// that loads a resource (`img`, `link`, `iframe`), runs code (`script`), @@ -540,6 +558,9 @@ enum Request { Open { id: u64, html: String, + /// Where Vello paints, straight. + paint: wgpu::TextureView, + /// Where the paint lands premultiplied, for the compositor to sample. view: wgpu::TextureView, start: Instant, }, @@ -575,27 +596,33 @@ impl Overlays { /// Lay out `html` at `width` by `height`, its CSS timeline starting at /// `start` on the program clock. The HTML has passed [`check`]. pub fn open(&mut self, html: &str, width: u32, height: u32, start: Instant) -> Overlay { - let texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("moq-scene overlay"), - size: wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - // Vello writes gamma-encoded straight alpha, what the compositor reads. - format: crate::FORMAT, - usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::TEXTURE_BINDING, - view_formats: &[], - }); + let texture = |label, usage| { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + // Gamma-encoded, as the compositor reads. + format: FORMAT, + usage: usage | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + }; + // Vello writes straight alpha, which one pass per paint premultiplies. + let paint = texture("moq-scene overlay paint", wgpu::TextureUsages::STORAGE_BINDING); + let texture = texture("moq-scene overlay", wgpu::TextureUsages::RENDER_ATTACHMENT); let id = self.next; self.next += 1; // A thread that stopped fails the next paint, which says so. let _ = self.requests.send(Request::Open { id, html: html.to_string(), + paint: paint.create_view(&Default::default()), view: texture.create_view(&Default::default()), start, }); @@ -640,7 +667,7 @@ impl Overlay { self.id } - /// The texture the overlay paints into, straight alpha. + /// The texture each paint lands in, premultiplied. pub fn texture(&self) -> &wgpu::Texture { &self.texture } @@ -664,7 +691,13 @@ fn run( let mut painter: Option> = None; for request in requests { match request { - Request::Open { id, html, view, start } => { + Request::Open { + id, + html, + paint, + view, + start, + } => { let size = view.texture().size(); let mut document = document(&html, size.width, size.height); // Stylo starts an animation at the time of the resolve that @@ -673,7 +706,9 @@ fn run( document.resolve(0.0); let document = Document { document, + paint, view, + premultiply: None, start, painted: false, }; @@ -705,11 +740,13 @@ fn run( } } -/// Vello's compute pipelines on the compositor's device, compiled once and -/// shared by every overlay. +/// Vello's compute pipelines on the compositor's device, and the pass that +/// premultiplies what they paint, compiled once and shared by every overlay. struct Painter { renderer: vello::Renderer, scene: vello::Scene, + premultiply: wgpu::RenderPipeline, + layout: wgpu::BindGroupLayout, } impl Painter { @@ -724,17 +761,107 @@ impl Painter { ..Default::default() }; let renderer = vello::Renderer::new(device, options).map_err(|err| err.to_string())?; + + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene premultiply"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene premultiply"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene premultiply"), + source: wgpu::ShaderSource::Wgsl(PREMULTIPLY.into()), + }); + let premultiply = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene premultiply"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); Ok(Self { renderer, scene: vello::Scene::new(), + premultiply, + layout, }) } + + /// Copy `paint` into `view`, premultiplied, after the work already queued. + fn premultiply( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + bind: &wgpu::BindGroup, + view: &wgpu::TextureView, + ) { + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene premultiply"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene premultiply"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&self.premultiply); + pass.set_bind_group(0, bind, &[]); + pass.draw(0..3, 0..1); + } + queue.submit([encoder.finish()]); + } } /// An overlay's document on the overlay thread, and where it paints. struct Document { document: BaseDocument, + /// Where Vello paints, straight. + paint: wgpu::TextureView, + /// Where the paint lands premultiplied. view: wgpu::TextureView, + /// `paint` bound for [`Painter::premultiply`], from the first paint on. + premultiply: Option, /// Where its CSS timeline starts on the program clock. start: Instant, /// Whether the texture holds a paint yet. @@ -757,7 +884,7 @@ impl Document { self.document .resolve(now.saturating_duration_since(self.start).as_secs_f64()); - let size = self.view.texture().size(); + let size = self.paint.texture().size(); painter.scene.reset(); let mut scene = anyrender_vello::VelloScenePainter::new(&mut painter.scene); blitz_paint::paint_scene(&mut scene, &mut self.document, 1.0, size.width, size.height, 0, 0); @@ -769,8 +896,19 @@ impl Document { }; painter .renderer - .render_to_texture(device, queue, &painter.scene, &self.view, ¶ms) + .render_to_texture(device, queue, &painter.scene, &self.paint, ¶ms) .map_err(|err| err.to_string())?; + let bind = self.premultiply.get_or_insert_with(|| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene premultiply"), + layout: &painter.layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&self.paint), + }], + }) + }); + painter.premultiply(device, queue, bind, &self.view); self.painted = true; Ok(()) }