diff --git a/end_to_end/tpu/run_wan_fast_inference.sh b/end_to_end/tpu/run_wan_fast_inference.sh index 567035a01..dffdaf0f4 100755 --- a/end_to_end/tpu/run_wan_fast_inference.sh +++ b/end_to_end/tpu/run_wan_fast_inference.sh @@ -68,7 +68,7 @@ # v6e seed 777, 17.92 dB / 0.6556 on v6e seed 12345, and # 17.69 dB / 0.6918 on tpu7x seed 12345 (visually clean). # USE_FUSED_ROPE_KERNEL=false restores the XLA producer. -# WAN_CROSS_ATTN_KERNEL +# WAN_CROSS_ATTN_KERNEL / WAN_PATCH_EMBED_MODE # override Wan fast-inference kernel/layout switches # DP / CP / PER_DEVICE_BATCH / SEED # override mesh parallelism, per-device batch, or RNG seed (default 12345) @@ -182,6 +182,7 @@ case "$TPU_PROFILE" in DEFAULT_VAE_SPATIAL=8 DEFAULT_FUSED_ROPE=true DEFAULT_CROSS_ATTN_KERNEL=pallas + DEFAULT_PATCH_EMBED_MODE=tokens ;; v7) PLATFORM_LIBTPU="$V7_LIBTPU" @@ -199,6 +200,7 @@ case "$TPU_PROFILE" in DEFAULT_VAE_SPATIAL=8 DEFAULT_FUSED_ROPE=true DEFAULT_CROSS_ATTN_KERNEL=pallas + DEFAULT_PATCH_EMBED_MODE=tokens ;; *) echo "== warning: unrecognised or non-v6e/v7 accelerator '${ACCEL_TYPE:-unknown}';" \ @@ -218,6 +220,7 @@ case "$TPU_PROFILE" in DEFAULT_VAE_SPATIAL=8 DEFAULT_FUSED_ROPE=false DEFAULT_CROSS_ATTN_KERNEL=xla + DEFAULT_PATCH_EMBED_MODE=tokens ;; esac @@ -250,6 +253,7 @@ FUSED_ROPE_ARGS=() [ -n "${FUSED_ROPE_BLOCK_S:-}" ] && FUSED_ROPE_ARGS+=("fused_rope_block_s=$FUSED_ROPE_BLOCK_S") [ -n "${FUSED_ROPE_HEAD_BLOCK:-}" ] && FUSED_ROPE_ARGS+=("fused_rope_head_block=$FUSED_ROPE_HEAD_BLOCK") WAN_CROSS_ATTN_KERNEL=${WAN_CROSS_ATTN_KERNEL:-$DEFAULT_CROSS_ATTN_KERNEL} +WAN_PATCH_EMBED_MODE=${WAN_PATCH_EMBED_MODE:-$DEFAULT_PATCH_EMBED_MODE} # Mesh: context parallelism carries the Ulysses shards, data parallelism takes # whatever devices remain. Defaults to CP=4 / DP=2 on an 8-device slice. On a @@ -317,6 +321,7 @@ python src/maxdiffusion/generate_wan.py "$CONFIG" \ use_fused_rope_kernel="$USE_FUSED_ROPE_KERNEL" \ ${FUSED_ROPE_ARGS[@]+"${FUSED_ROPE_ARGS[@]}"} \ wan_cross_attn_kernel="$WAN_CROSS_ATTN_KERNEL" \ + wan_patch_embed_mode="$WAN_PATCH_EMBED_MODE" \ fps=16 ${GUIDANCE_ARGS[@]+"${GUIDANCE_ARGS[@]}"} \ seed="${SEED:-12345}" \ flash_block_sizes="$FLASH_BLOCK_SIZES" \ diff --git a/end_to_end/tpu/run_wan_stack_tests.sh b/end_to_end/tpu/run_wan_stack_tests.sh index b5dce949d..1a2fc874e 100755 --- a/end_to_end/tpu/run_wan_stack_tests.sh +++ b/end_to_end/tpu/run_wan_stack_tests.sh @@ -47,5 +47,7 @@ TESTS=( "$T/wan_runtime_options_test.py" # Short-KV cross-attention Pallas kernel (feat/wan-cross-attn-pallas) "$T/cross_attention_pallas_test.py" + # Token-sharded patch embedding (feat/wan-token-patch-embed) + "$T/wan_patch_embed_test.py" ) PYTHONPATH="src${PYTHONPATH:+:$PYTHONPATH}" exec "${PYTHON:-python3}" -m pytest -q -rs "${TESTS[@]}" "$@" diff --git a/src/maxdiffusion/configs/base_wan_14b.yml b/src/maxdiffusion/configs/base_wan_14b.yml index 100c730af..fcffe36dd 100644 --- a/src/maxdiffusion/configs/base_wan_14b.yml +++ b/src/maxdiffusion/configs/base_wan_14b.yml @@ -136,6 +136,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/configs/base_wan_1_3b.yml b/src/maxdiffusion/configs/base_wan_1_3b.yml index 804ef7f6b..a0ac148d2 100644 --- a/src/maxdiffusion/configs/base_wan_1_3b.yml +++ b/src/maxdiffusion/configs/base_wan_1_3b.yml @@ -133,6 +133,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/configs/base_wan_27b.yml b/src/maxdiffusion/configs/base_wan_27b.yml index a9e459f25..b0d4effa9 100644 --- a/src/maxdiffusion/configs/base_wan_27b.yml +++ b/src/maxdiffusion/configs/base_wan_27b.yml @@ -170,6 +170,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring*, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/configs/base_wan_animate.yml b/src/maxdiffusion/configs/base_wan_animate.yml index d008e54e3..d29872cee 100644 --- a/src/maxdiffusion/configs/base_wan_animate.yml +++ b/src/maxdiffusion/configs/base_wan_animate.yml @@ -115,6 +115,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/configs/base_wan_i2v_14b.yml b/src/maxdiffusion/configs/base_wan_i2v_14b.yml index e0cbfa79b..a4d88d888 100644 --- a/src/maxdiffusion/configs/base_wan_i2v_14b.yml +++ b/src/maxdiffusion/configs/base_wan_i2v_14b.yml @@ -117,6 +117,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/configs/base_wan_i2v_27b.yml b/src/maxdiffusion/configs/base_wan_i2v_27b.yml index a53d27e51..53936ead3 100644 --- a/src/maxdiffusion/configs/base_wan_i2v_27b.yml +++ b/src/maxdiffusion/configs/base_wan_i2v_27b.yml @@ -117,6 +117,10 @@ wan_rope_accum: "auto" # (KV shorter than flash_min_seq_length): "xla", or "pallas" for the fused kernel that # keeps the KV in VMEM (within bf16 rounding of "xla", not bit-identical). wan_cross_attn_kernel: "xla" +# Patch embedding: "conv" (nnx.Conv; its channel-sharded output is resharded by +# token, an all-gather on tpu7x) or "tokens" (token-sharded patches times the +# replicated kernel, no activation collective; equal up to accumulation order). +wan_patch_embed_mode: "conv" # For attention=ulysses_ring, hidden Ulysses shard count; ring shards are context / this. ulysses_shards: -1 # Splits Ulysses all-to-all into head-group chunks. The last chunk carries any remainder. diff --git a/src/maxdiffusion/models/wan/transformers/transformer_wan.py b/src/maxdiffusion/models/wan/transformers/transformer_wan.py index db083ae0c..53e14a2ca 100644 --- a/src/maxdiffusion/models/wan/transformers/transformer_wan.py +++ b/src/maxdiffusion/models/wan/transformers/transformer_wan.py @@ -61,6 +61,62 @@ def get_frequencies(max_seq_len: int, theta: int, attention_head_dim: int): return freqs_split +PATCH_EMBED_MODES = ("conv", "tokens") + + +def can_embed_patches_as_tokens(conv: nnx.Conv, hidden_shape: Tuple[int, ...], patch_size: Tuple[int, int, int]) -> bool: + """Whether `embed_patches_as_tokens` computes exactly what `conv` does.""" + + def per_dim(value): + value = 1 if value is None else value + return tuple(value) if isinstance(value, (tuple, list)) else (value,) * len(patch_size) + + if len(hidden_shape) != 5 or per_dim(conv.kernel_size) != tuple(patch_size): + return False + if per_dim(conv.strides) != tuple(patch_size) or conv.feature_group_count != 1 or conv.mask is not None: + return False + if per_dim(conv.input_dilation) != (1,) * 3 or per_dim(conv.kernel_dilation) != (1,) * 3: + return False + # With every spatial size divisible by its stride, 'SAME' pads nothing. + return conv.padding in ("SAME", "VALID") and all(n % p == 0 for n, p in zip(hidden_shape[1:4], patch_size)) + + +def embed_patches_as_tokens(conv: nnx.Conv, hidden_states: jax.Array, patch_size: Tuple[int, int, int]) -> jax.Array: + """Wan patch embedding as a token-sharded matmul: `[B, F, H, W, C]` -> `[B, N, D]`. + + Same result as `jax.lax.collapse(conv(hidden_states), 1, -1)` up to + accumulation order, but produced in the layout the transformer blocks use. + When the 3D `nnx.Conv` runs before `collapse`, XLA either shards the conv output + differently from `activation_length` or emits an activation all-gather / + collective-permute when collapsing `(F, H, W)` into the context-sharded token + axis (12.8 ms per step on tpu7x, exposed). Patches do not overlap (stride == + kernel), so each token needs only its own patch: flatten the patches in the + kernel's `(p_t, p_h, p_w, C)` order, shard them by token and multiply by the + replicated `[p_t*p_h*p_w*C, D]` kernel. + """ + p_t, p_h, p_w = patch_size + batch, frames, height, width, channels = hidden_states.shape + patches = hidden_states.reshape(batch, frames // p_t, p_t, height // p_h, p_h, width // p_w, p_w, channels) + patches = jnp.transpose(patches, (0, 1, 3, 5, 2, 4, 6, 7)) + patches = patches.reshape(batch, -1, p_t * p_h * p_w * channels) + patches = nn.with_logical_constraint(patches, ("activation_batch", "activation_length", None)) + + kernel = conv.kernel[...].reshape(p_t * p_h * p_w * channels, -1) + kernel = nn.with_logical_constraint(kernel, (None, None)) + bias = conv.bias[...] if conv.bias is not None else None + # Mirror nnx.Conv: promote everything to `conv.dtype`, no preferred type. + dtype = conv.dtype or jnp.result_type(patches, kernel) + out = jax.lax.dot_general( + patches.astype(dtype), + kernel.astype(dtype), + (((2,), (0,)), ((), ())), + precision=conv.precision, + ) + if bias is not None: + out = out + bias.astype(dtype) + return out + + class WanRotaryPosEmbed(nnx.Module): def __init__( @@ -609,6 +665,13 @@ def __init__( **(attention_config or {}), } + # Resolved at build time so they live on the GraphDef; None -> built-in default. + def _wan_opt(name): + value = attention_config.get(name) + return wan_runtime_options.default(name) if value is None else wan_runtime_options.coerce(name, value) + + self.patch_embed_mode = _wan_opt("wan_patch_embed_mode") + # 1. Patch & position embedding self.rope = WanRotaryPosEmbed(attention_head_dim, patch_size, rope_max_seq_len) self.patch_embedding = nnx.Conv( @@ -810,8 +873,19 @@ def __call__( if rotary_emb is None: rotary_emb = self.rope(hidden_states) with self.conditional_named_scope("patch_embedding"): - hidden_states = self.patch_embedding(hidden_states) - hidden_states = jax.lax.collapse(hidden_states, 1, -1) + patch_embed_mode = self.patch_embed_mode + if patch_embed_mode not in PATCH_EMBED_MODES: + raise ValueError(f"wan_patch_embed_mode must be one of {PATCH_EMBED_MODES}, got {patch_embed_mode!r}.") + if patch_embed_mode == "tokens": + if not can_embed_patches_as_tokens(self.patch_embedding, hidden_states.shape, self.config.patch_size): + raise ValueError( + f"wan_patch_embed_mode='tokens' requires the latent grid {hidden_states.shape[1:4]} to be divisible by " + f"patch_size {tuple(self.config.patch_size)} (height/width multiples of {8 * p_h}/{8 * p_w} pixels)." + ) + hidden_states = embed_patches_as_tokens(self.patch_embedding, hidden_states, self.config.patch_size) + else: + hidden_states = self.patch_embedding(hidden_states) + hidden_states = jax.lax.collapse(hidden_states, 1, -1) per_token_t = timestep.ndim == 2 # [B, seq_len] for TI2V with self.conditional_named_scope("condition_embedder"): if per_token_t: diff --git a/src/maxdiffusion/tests/wan_patch_embed_test.py b/src/maxdiffusion/tests/wan_patch_embed_test.py new file mode 100644 index 000000000..9e04a5cbe --- /dev/null +++ b/src/maxdiffusion/tests/wan_patch_embed_test.py @@ -0,0 +1,182 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""Tests for the token-sharded Wan patch embedding (`wan_patch_embed_mode`).""" + +import os +import unittest +from unittest import mock + +import jax +import jax.numpy as jnp +import numpy as np +from flax import nnx +from flax.linen import partitioning as nn_partitioning +from jax.sharding import Mesh + +from maxdiffusion import pyconfig +from maxdiffusion.max_utils import create_device_mesh +from maxdiffusion.models.wan.transformers import transformer_wan +from maxdiffusion.models.wan.transformers.transformer_wan import ( + WanModel, + can_embed_patches_as_tokens, + embed_patches_as_tokens, +) + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _conv(patch_size, in_channels=4, out_channels=256, dtype=jnp.float32, precision=jax.lax.Precision.HIGHEST, **kw): + return nnx.Conv( + in_channels, + out_channels, + kernel_size=patch_size, + strides=patch_size, + dtype=dtype, + param_dtype=dtype, + precision=precision, + rngs=nnx.Rngs(0), + bias_init=nnx.initializers.normal(0.5), + **kw, + ) + + +class PatchEmbedTokensTest(unittest.TestCase): + + def setUp(self): + super().setUp() + pyconfig.initialize([None, os.path.join(THIS_DIR, "..", "configs", "base_wan_14b.yml")], unittest=True) + self.config = pyconfig.config + self.mesh = Mesh(create_device_mesh(self.config), self.config.mesh_axes) + + def _both(self, conv, x, patch_size): + def conv_path(x): + return jax.lax.collapse(conv(x), 1, -1) + + def token_path(x): + return embed_patches_as_tokens(conv, x, patch_size) + + with self.mesh, nn_partitioning.axis_rules(self.config.logical_axis_rules): + return jax.jit(conv_path)(x), jax.jit(token_path)(x) + + def test_matches_conv(self): + for patch_size in ((1, 2, 2), (2, 2, 2)): + with self.subTest(patch_size=patch_size): + conv = _conv(patch_size) + x = jax.random.normal(jax.random.key(1), (2, 4, 8, 16, 4), jnp.float32) + want, got = self._both(conv, x, patch_size) + self.assertEqual(got.shape, want.shape) + np.testing.assert_allclose(np.asarray(got), np.asarray(want), rtol=1e-5, atol=1e-5) + + def test_matches_conv_in_bf16(self): + patch_size = (1, 2, 2) + conv = _conv(patch_size, in_channels=16, dtype=jnp.bfloat16, precision=None) + x = jax.random.normal(jax.random.key(2), (2, 3, 8, 16, 16), jnp.float32).astype(jnp.bfloat16) + want, got = self._both(conv, x, patch_size) + self.assertEqual(got.dtype, want.dtype) + np.testing.assert_allclose(np.asarray(got, np.float32), np.asarray(want, np.float32), rtol=2e-2, atol=2e-2) + + def test_guard(self): + conv = _conv((1, 2, 2)) + self.assertTrue(can_embed_patches_as_tokens(conv, (1, 21, 90, 160, 4), (1, 2, 2))) + # A spatial size not divisible by the patch makes 'SAME' pad. + self.assertFalse(can_embed_patches_as_tokens(conv, (1, 21, 91, 160, 4), (1, 2, 2))) + self.assertFalse(can_embed_patches_as_tokens(conv, (1, 21, 90, 160, 4), (1, 1, 2))) + overlapping = nnx.Conv(4, 8, kernel_size=(1, 2, 2), strides=(1, 1, 1), rngs=nnx.Rngs(0)) + self.assertFalse(can_embed_patches_as_tokens(overlapping, (1, 21, 90, 160, 4), (1, 2, 2))) + dilated = _conv((1, 2, 2), kernel_dilation=(1, 2, 2)) + self.assertFalse(can_embed_patches_as_tokens(dilated, (1, 21, 90, 160, 4), (1, 2, 2))) + + +class WanModelPatchEmbedModeTest(unittest.TestCase): + + def setUp(self): + super().setUp() + pyconfig.initialize([None, os.path.join(THIS_DIR, "..", "configs", "base_wan_14b.yml")], unittest=True) + self.config = pyconfig.config + self.mesh = Mesh(create_device_mesh(self.config), self.config.mesh_axes) + with nn_partitioning.axis_rules(self.config.logical_axis_rules): + self.model = WanModel( + rngs=nnx.Rngs(0), + num_attention_heads=2, + attention_head_dim=128, + in_channels=4, + out_channels=4, + text_dim=32, + freq_dim=32, + ffn_dim=256, + num_layers=1, + mesh=self.mesh, + attention="dot_product", + ) + self.inputs = { + "hidden_states": jax.random.normal(jax.random.key(3), (1, 4, 2, 8, 16), jnp.float32), + "timestep": jnp.ones((1,)), + "encoder_hidden_states": jax.random.normal(jax.random.key(4), (1, 16, 32), jnp.float32), + } + + def _run(self, mode): + # The mode is a build-time attribute (normally from attention_config). + self.model.patch_embed_mode = mode + with self.mesh, nn_partitioning.axis_rules(self.config.logical_axis_rules): + return self.model(**self.inputs) + + def test_tokens_mode_uses_the_token_path_and_matches_conv(self): + want = self._run("conv") + with mock.patch.object(transformer_wan, "embed_patches_as_tokens", wraps=transformer_wan.embed_patches_as_tokens) as spy: + got = self._run("tokens") + spy.assert_called_once() + self.assertEqual(got.shape, want.shape) + np.testing.assert_allclose(np.asarray(got), np.asarray(want), rtol=2e-2, atol=2e-2) + + def test_conv_mode_does_not_use_the_token_path(self): + with mock.patch.object(transformer_wan, "embed_patches_as_tokens") as spy: + self._run("conv") + spy.assert_not_called() + + def test_tokens_mode_raises_on_indivisible_latent_grid(self): + self.model.patch_embed_mode = "tokens" + odd_inputs = {**self.inputs, "hidden_states": jnp.zeros((1, 4, 2, 7, 16), jnp.float32)} + with self.mesh, nn_partitioning.axis_rules(self.config.logical_axis_rules): + with self.assertRaisesRegex(ValueError, "wan_patch_embed_mode='tokens' requires the latent grid"): + self.model(**odd_inputs) + + def test_rejects_unknown_mode(self): + with self.assertRaises(ValueError): + self._run("bogus") + + def test_mode_comes_from_attention_config_not_env(self): + with mock.patch.dict(os.environ, {"WAN_PATCH_EMBED_MODE": "tokens"}): + with nn_partitioning.axis_rules(self.config.logical_axis_rules): + model = nnx.eval_shape( + lambda: WanModel( + rngs=nnx.Rngs(0), + num_attention_heads=2, + attention_head_dim=128, + in_channels=4, + out_channels=4, + text_dim=32, + freq_dim=32, + ffn_dim=256, + num_layers=1, + attention="dot_product", + ) + ) + self.assertEqual(model.patch_embed_mode, "conv") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/maxdiffusion/tests/wan_runtime_options_test.py b/src/maxdiffusion/tests/wan_runtime_options_test.py index cd401105d..cb040eea1 100644 --- a/src/maxdiffusion/tests/wan_runtime_options_test.py +++ b/src/maxdiffusion/tests/wan_runtime_options_test.py @@ -134,7 +134,9 @@ def test_unset_options_resolve_to_defaults_not_env(self): } with mock.patch.dict(os.environ, env): attn = self._attn() - for name in opts.ATTENTION_OPTIONS: + # wan_patch_embed_mode rides in attention_config too, but is consumed by + # WanModel (covered in wan_patch_embed_test). + for name in sorted(set(opts.ATTENTION_OPTIONS) - {"wan_patch_embed_mode"}): with self.subTest(option=name): self.assertEqual(getattr(attn, name), opts.default(name)) @@ -182,6 +184,7 @@ def test_wan_model_resolves_cfg_before_unpatchify(self): def test_enum_options_validation(self): valid_cases = { "wan_cross_attn_kernel": ("xla", "pallas"), + "wan_patch_embed_mode": ("conv", "tokens"), } for name, valid_values in valid_cases.items(): for val in valid_values: diff --git a/src/maxdiffusion/utils/wan_runtime_options.py b/src/maxdiffusion/utils/wan_runtime_options.py index c91b1c1a9..d540acd9e 100644 --- a/src/maxdiffusion/utils/wan_runtime_options.py +++ b/src/maxdiffusion/utils/wan_runtime_options.py @@ -43,6 +43,7 @@ "wan_cross_attn_prescale_kv": (False, "bool"), "wan_rope_accum": ("auto", "str"), "wan_cross_attn_kernel": ("xla", "str"), + "wan_patch_embed_mode": ("conv", "str"), } @@ -64,6 +65,8 @@ def _coerce(name: str, value: Any) -> Any: raise ValueError(f"wan_rope_accum must be 'auto', 'dtype', or 'f32', got {value!r}.") if name == "wan_cross_attn_kernel" and val not in ("xla", "pallas"): raise ValueError(f"wan_cross_attn_kernel must be 'xla' or 'pallas', got {value!r}.") + if name == "wan_patch_embed_mode" and val not in ("conv", "tokens"): + raise ValueError(f"wan_patch_embed_mode must be 'conv' or 'tokens', got {value!r}.") return val @@ -105,6 +108,7 @@ def resolve_from_config(config: Any, name: str) -> Any: "wan_cross_attn_prescale_kv", "wan_rope_accum", "wan_cross_attn_kernel", + "wan_patch_embed_mode", )