From 46c1ae2f0be53a3a15997782ed6ff735c2b2149b Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Tue, 17 Nov 2026 19:09:25 +0800 Subject: [PATCH 1/4] fix(attention): support native cuDNN MHA64 with eager rounding --- aspec/kernels/cutedsl/dspark_qk_norm_rope.py | 28 +- .../cutedsl/dspark_qk_norm_rope_bwd.py | 48 +++- aspec/models/draft/dflash.py | 15 +- aspec/models/ops/cudnn_flex_attention.py | 8 +- aspec/models/ops/dspark_qk_norm_rope.py | 2 + docs/user-guide/structures.md | 29 +++ tests/cpu/test_cudnn_attention_geometry.py | 28 ++ tests/test_cudnn_mha64.py | 239 ++++++++++++++++++ 8 files changed, 377 insertions(+), 20 deletions(-) create mode 100644 tests/cpu/test_cudnn_attention_geometry.py create mode 100644 tests/test_cudnn_mha64.py diff --git a/aspec/kernels/cutedsl/dspark_qk_norm_rope.py b/aspec/kernels/cutedsl/dspark_qk_norm_rope.py index 9285279..1d27c62 100644 --- a/aspec/kernels/cutedsl/dspark_qk_norm_rope.py +++ b/aspec/kernels/cutedsl/dspark_qk_norm_rope.py @@ -2,12 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 # # Adapted by ASpec Authors from NVIDIA cuDNN Frontend cb8cec8. +# ASpec modifications Copyright (c) 2026 Ant Group Co., Ltd. """ASpec-owned CuTeDSL kernel for DSpark Q/K RMSNorm and indexed RoPE. The kernel reads draft Q plus separate context/draft K planes and writes compact Q and logical ``[context, draft]`` K outputs without materializing a K concat. -RMSNorm and RoPE arithmetic stay in FP32 and round once to the input dtype. +The legacy path rounds once after FP32 normalization and rotation. The optional +intermediate-rounding mode preserves the eager input-dtype arithmetic boundaries. +Normalization reductions remain FP32 in both modes. """ from typing import NamedTuple, Optional @@ -124,6 +127,7 @@ def frost_qk_norm_rope( const_head_counts: cutlass.Constexpr[bool], batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], + round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """``rows_per_group`` consecutive rows per lane group: ALL loads issued first, then the reductions and stores. @@ -355,7 +359,13 @@ def frost_qk_norm_rope( ys = [] for c in cutlass.range_constexpr(chunks): if cutlass.const_expr(apply_norm): - ys.append([xs[r][c][i] * rstd * ws[r][c][i] for i in range(ELEMS_PER_ACCESS)]) + if cutlass.const_expr(round_intermediates): + normalized = [(xs[r][c][i] * rstd).to(mQ.element_type).to(cutlass.Float32) + for i in range(ELEMS_PER_ACCESS)] + ys.append([(normalized[i] * ws[r][c][i]).to(mQ.element_type).to(cutlass.Float32) + for i in range(ELEMS_PER_ACCESS)]) + else: + ys.append([xs[r][c][i] * rstd * ws[r][c][i] for i in range(ELEMS_PER_ACCESS)]) else: ys.append(list(xs[r][c])) @@ -368,7 +378,12 @@ def frost_qk_norm_rope( for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): partner = cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, ys[0][i], cutlass.Int32(half_lanes), 31, kind=nvvm.Shfl.BFLY)) signed = -partner if lane < cutlass.Int32(half_lanes) else partner - rotated.append(cute.math.fma(ys[0][i], cs[r][i], signed * ss[r][i])) + if cutlass.const_expr(round_intermediates): + direct = (ys[0][i] * cs[r][i]).to(mQ.element_type).to(cutlass.Float32) + other = (signed * ss[r][i]).to(mQ.element_type).to(cutlass.Float32) + rotated.append(direct + other) + else: + rotated.append(cute.math.fma(ys[0][i], cs[r][i], signed * ss[r][i])) for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): ys[0][i] = rotated[i] if in_rope else ys[0][i] @@ -457,6 +472,7 @@ def qk_norm_rope_launch( batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], stream: cuda.CUstream, + round_intermediates: cutlass.Constexpr[bool] = False, ): frost_qk_norm_rope( q, @@ -490,6 +506,7 @@ def qk_norm_rope_launch( const_head_counts, batch_one, use_pdl, + round_intermediates, ).launch(grid=(n_blocks, 1, 1), block=(threads_per_cta, 1, 1), stream=stream, use_pdl=use_pdl) @@ -512,6 +529,7 @@ class DSparkQkNormRopeRecipe(NamedTuple): eps: float rows_per_cta: int batch_one: bool = False + round_intermediates: bool = False def _fake(dtype, shape, stride_order): @@ -538,6 +556,7 @@ def compile_dspark_qk_norm_rope( const_head_counts: bool = DEFAULT_CONST_HEAD_COUNTS, batch_one: bool = False, use_pdl: bool = False, + round_intermediates: bool = False, ) -> DSparkQkNormRopeRecipe: """Compile the DSpark asymmetric, indexed-RoPE training specialization.""" @@ -557,6 +576,7 @@ def compile_dspark_qk_norm_rope( bool(const_head_counts), bool(batch_one), bool(use_pdl), + bool(round_intermediates), torch.cuda.current_device(), ) if key not in compiled_cache: @@ -606,6 +626,7 @@ def compile_dspark_qk_norm_rope( bool(batch_one), bool(use_pdl), _FAKE_STREAM, + bool(round_intermediates), options="--enable-tvm-ffi", ) return DSparkQkNormRopeRecipe( @@ -617,6 +638,7 @@ def compile_dspark_qk_norm_rope( eps=float(eps), rows_per_cta=(threads_per_cta // lanes_per_row(d)) * rows_per_group, batch_one=bool(batch_one), + round_intermediates=bool(round_intermediates), ) diff --git a/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py b/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py index b3f92ab..42e5472 100644 --- a/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py +++ b/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Adapted by ASpec Authors from NVIDIA cuDNN Frontend cb8cec8. +# ASpec modifications Copyright (c) 2026 Ant Group Co., Ltd. """ASpec-owned CuTeDSL adjoint for the DSpark Q/K preparation kernel. @@ -131,6 +132,7 @@ def _row_class( mAmaxOut: Optional[cute.Tensor], # [>= grid] fp32: partials[cta] = max |stored words| of this CTA (plain store at CTA end); None = no fold cta: cutlass.Int32, # the GRID-level block index: the partial this CTA owns sred, # the ONE sRed SMEM Array of the kernel, fp32 [warps] (None without want_amax) + round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """The persistent loop of ONE row class (Q, K or V) for one CTA. @@ -288,9 +290,15 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one rotated = [] for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): t_i = g[0][i] * sin_v[i] + if cutlass.const_expr(round_intermediates): + t_i = t_i.to(io_dtype).to(cutlass.Float32) partner = cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, t_i, cutlass.Int32(half_lanes), 31, kind=nvvm.Shfl.BFLY)) signed = partner if lane < cutlass.Int32(half_lanes) else -partner # the forward's sign, FLIPPED - rotated.append(cute.math.fma(g[0][i], cos_v[i], signed)) + if cutlass.const_expr(round_intermediates): + direct = (g[0][i] * cos_v[i]).to(io_dtype).to(cutlass.Float32) + rotated.append((direct + signed).to(io_dtype).to(cutlass.Float32)) + else: + rotated.append(cute.math.fma(g[0][i], cos_v[i], signed)) for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): g[0][i] = rotated[i] if in_rope else g[0][i] @@ -304,6 +312,9 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one rstd = rstds[r] x_hat = [[xs[r][c][i] * rstd for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] gw = [[g[c][i] * ws[c][i] for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] + if cutlass.const_expr(round_intermediates): + gw = [[gw[c][i].to(io_dtype).to(cutlass.Float32) + for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] acc = zero for c in cutlass.range_constexpr(chunks): for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): @@ -317,15 +328,23 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one keep = valid & (~deads[r]) if cutlass.const_expr(has_seq_lens) else valid for c in cutlass.range_constexpr(chunks): for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): - dw_acc[c * ELEMS_PER_ACCESS + i] = ( - cute.math.fma( - g[c][i], - x_hat[c][i], - dw_acc[c * ELEMS_PER_ACCESS + i], + if cutlass.const_expr(round_intermediates): + normalized = x_hat[c][i].to(io_dtype).to(cutlass.Float32) + contribution = (g[c][i] * normalized).to(io_dtype).to(cutlass.Float32) + dw_acc[c * ELEMS_PER_ACCESS + i] = ( + dw_acc[c * ELEMS_PER_ACCESS + i] + contribution + if keep else dw_acc[c * ELEMS_PER_ACCESS + i] + ) + else: + dw_acc[c * ELEMS_PER_ACCESS + i] = ( + cute.math.fma( + g[c][i], + x_hat[c][i], + dw_acc[c * ELEMS_PER_ACCESS + i], + ) + if keep + else dw_acc[c * ELEMS_PER_ACCESS + i] ) - if keep - else dw_acc[c * ELEMS_PER_ACCESS + i] - ) else: # RoPE-only: dx IS g; [rope_dim, D) goes back through fp32_to_fp16 # untouched by any fp32 op -> a bit-exact copy (asserted by the tests). @@ -437,6 +456,7 @@ def frost_qk_norm_rope_bwd( rows_per_group: cutlass.Constexpr[int], batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], + round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """Three CTA classes over one grid (Q | K | V), each a persistent row loop; see the module docstring.""" if cutlass.const_expr(use_pdl): @@ -524,6 +544,7 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, + round_intermediates, ) else: if cta < n_ctas_q + n_ctas_k: @@ -585,6 +606,7 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, + round_intermediates, ) else: # ---- V class: the band copy (no norm, no RoPE, no dW) -------------------- @@ -638,6 +660,7 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, + round_intermediates, ) if cutlass.const_expr(use_pdl): @@ -771,6 +794,7 @@ def qk_norm_rope_bwd_launch( batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], stream: cuda.CUstream, + round_intermediates: cutlass.Constexpr[bool] = False, ): frost_qk_norm_rope_bwd( dq, @@ -815,6 +839,7 @@ def qk_norm_rope_bwd_launch( rows_per_group, batch_one, use_pdl, + round_intermediates, ).launch(grid=(n_blocks, 1, 1), block=(threads_per_cta, 1, 1), stream=stream, use_pdl=use_pdl) @@ -874,6 +899,7 @@ class QkNormRopeBwdRecipe(NamedTuple): # Appended (default = today's artifact): the amax fold of the stored bands as per-CTA partials (``amax_out`` REQUIRED). want_amax: bool = False batch_one: bool = False + round_intermediates: bool = False def _resolve_n_ctas_cap(policy: str, device: int) -> int: @@ -905,6 +931,7 @@ def compile_dspark_qk_norm_rope_bwd( batch_one: bool = False, use_pdl: bool = False, n_ctas_policy: str = DEFAULT_N_CTAS_POLICY, + round_intermediates: bool = False, ) -> QkNormRopeBwdRecipe: """Compile the DSpark asymmetric, indexed-RoPE training adjoint. @@ -939,6 +966,7 @@ def compile_dspark_qk_norm_rope_bwd( bool(const_head_counts), bool(batch_one), bool(use_pdl), + bool(round_intermediates), str(n_ctas_policy), device, ) @@ -1007,6 +1035,7 @@ def compile_dspark_qk_norm_rope_bwd( bool(batch_one), bool(use_pdl), _FAKE_STREAM, + bool(round_intermediates), options="--enable-tvm-ffi", ) reduce_key = (int(d), device) @@ -1044,6 +1073,7 @@ def compile_dspark_qk_norm_rope_bwd( rows_per_group=int(rows_per_group), want_amax=False, batch_one=bool(batch_one), + round_intermediates=bool(round_intermediates), ) diff --git a/aspec/models/draft/dflash.py b/aspec/models/draft/dflash.py index 5408a24..1ec132e 100644 --- a/aspec/models/draft/dflash.py +++ b/aspec/models/draft/dflash.py @@ -422,11 +422,16 @@ def set_attention_backend(self, attention_backend: str) -> None: "DFlash GQA attention supports flex_attention and " f"cudnn_flex_attention, got {attention_backend!r}" ) - if attention_backend == "cudnn_flex_attention" and self.head_dim != 128: - raise NotImplementedError( - "DFlash cuDNN FlexAttention currently requires head_dim=128, " - f"got {self.head_dim}" - ) + if attention_backend == "cudnn_flex_attention": + if self.head_dim not in {64, 128}: + raise NotImplementedError( + "DFlash cuDNN FlexAttention requires head_dim=64 or 128, " + f"got {self.head_dim}" + ) + if self.head_dim == 64 and self.num_heads != self.num_kv_heads: + raise NotImplementedError( + "DFlash cuDNN head_dim=64 currently requires Hq=Hkv (MHA)" + ) self.attention_backend = attention_backend def forward( diff --git a/aspec/models/ops/cudnn_flex_attention.py b/aspec/models/ops/cudnn_flex_attention.py index 71b9236..968bac8 100644 --- a/aspec/models/ops/cudnn_flex_attention.py +++ b/aspec/models/ops/cudnn_flex_attention.py @@ -27,7 +27,7 @@ import torch -_SUPPORTED_HEAD_DIMS = {(128, 128), (192, 128)} +_SUPPORTED_HEAD_DIMS = {(64, 64), (128, 128), (192, 128)} def _document_run_starts(ctx_doc_ids: torch.Tensor) -> torch.Tensor: @@ -373,9 +373,11 @@ def cudnn_flex_attention( dims = (query.shape[-1], value.shape[-1]) if dims not in _SUPPORTED_HEAD_DIMS: raise ValueError( - "the ASpec cuDNN path currently supports (Dqk,Dv)=(128,128) " - f"for GQA and (192,128) for materialized MLA; got {dims}" + "the ASpec cuDNN path currently supports (Dqk,Dv)=(64,64) for MHA, " + f"(128,128) for GQA and (192,128) for materialized MLA; got {dims}" ) + if dims == (64, 64) and query_heads != key_heads: + raise ValueError("cuDNN head_dim=64 currently requires Hq=Hkv (MHA)") if dims == (192, 128) and query_heads != key_heads: raise ValueError("materialized MLA currently requires Hq=Hkv") diff --git a/aspec/models/ops/dspark_qk_norm_rope.py b/aspec/models/ops/dspark_qk_norm_rope.py index d6cfefa..3ee19a3 100644 --- a/aspec/models/ops/dspark_qk_norm_rope.py +++ b/aspec/models/ops/dspark_qk_norm_rope.py @@ -92,6 +92,7 @@ def _forward_recipe( eps=eps, defer_secondary_loads=False, batch_one=batch_one, + round_intermediates=d == 64 and h_q == h_kv, ) _FWD_RECIPES[key] = recipe return recipe @@ -118,6 +119,7 @@ def _backward_recipe( rope_dim=d, eps=eps, batch_one=batch_one, + round_intermediates=d == 64 and h_q == h_kv, ) _BWD_RECIPES[key] = recipe return recipe diff --git a/docs/user-guide/structures.md b/docs/user-guide/structures.md index 704c415..8c75e4b 100644 --- a/docs/user-guide/structures.md +++ b/docs/user-guide/structures.md @@ -26,6 +26,35 @@ optimizer/FP32 master, gradients, activations and transfer buffers when checking resume/export preserves expert count, top-k, normalization, auxiliary policy and control state; missing or unverifiable routing state is not silently accepted. +## cuDNN attention geometry + +Set `training.attention_backend=cudnn_flex_attention` to select the cuDNN Frontend +backend in the [pinned runtime](installation.md). On SM90 and SM100, native MHA +supports query/key and value head widths of 64 with equal query and KV head counts. +The adapter also supports width-128 GQA and materialized MLA with query/key width +192 and value width 128. Width-64 GQA is outside the supported contract. + +The optional internal joint-GQA backward has a separate contract: head width 128 +and four query heads per KV head. Native MHA64 does not use that specialization. +Its fused Q/K normalization and RoPE preserve the eager FP16/BF16 intermediate +rounding boundaries while retaining FP32 normalization reductions. Keep the +attention backend fixed when reproducing or resuming a training run: identical-state +loss/gradient comparisons and independent optimizer trajectories are different checks. + +Run the MHA64 and retained attention/kernel regressions in the Linux Draft environment: + +```bash +ASPEC_RUN_CUDNN_FLEX_INTEGRATION=1 ASPEC_RUN_CUTEDSL_INTEGRATION=1 \ + bash tools/run_in_env.sh python -m pytest -q \ + tests/test_cudnn_mha64.py tests/test_cudnn_flex_attention.py tests/test_dspark_qk_cutedsl.py +``` + +The MHA64 training tests use synthetic features and bounded hidden/vocabulary sizes. +Add `ASPEC_RUN_CUDNN_MHA64_FULL=1` to include hidden size 4096, vocabulary size 154880 +and 64 anchors at both 8K and 256K. The full-width 256K case enables gradient checkpointing +uniformly across FA4, FlexAttention and cuDNN. These numerical checks do not establish +real-checkpoint training quality or convergence. + ## Packing and context parallelism ASpec supports two dense attention layouts. The padded-document layout keeps one diff --git a/tests/cpu/test_cudnn_attention_geometry.py b/tests/cpu/test_cudnn_attention_geometry.py new file mode 100644 index 0000000..0fc5f32 --- /dev/null +++ b/tests/cpu/test_cudnn_attention_geometry.py @@ -0,0 +1,28 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Offline geometry contracts for DFlash's cuDNN attention adapter.""" + +import pytest +import torch + +from aspec.models.draft.dflash import DFlashAttention, DFlashConfig + + +@pytest.mark.parametrize("head_dim,kv_heads,supported", [ + (64, 4, True), (64, 1, False), (96, 4, False), (128, 1, True), +]) +def test_cudnn_draft_head_geometry(head_dim, kv_heads, supported): + config = DFlashConfig( + hidden_size=256, intermediate_size=512, num_hidden_layers=1, + num_attention_heads=4, num_key_value_heads=kv_heads, + head_dim=head_dim, vocab_size=256, num_target_layers=1, + target_hidden_size=256, target_num_hidden_layers=1, + ) + with torch.device("meta"): + attention = DFlashAttention(config) + if supported: + attention.set_attention_backend("cudnn_flex_attention") + assert attention.attention_backend == "cudnn_flex_attention" + else: + with pytest.raises(NotImplementedError, match="head_dim"): + attention.set_attention_backend("cudnn_flex_attention") diff --git a/tests/test_cudnn_mha64.py b/tests/test_cudnn_mha64.py new file mode 100644 index 0000000..1362596 --- /dev/null +++ b/tests/test_cudnn_mha64.py @@ -0,0 +1,239 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""MHA64 DSpark loss and update parity across attention backends.""" + +import gc +import os + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUDNN_FLEX_INTEGRATION") != "1" or not torch.cuda.is_available(), + reason="requires a CUDA cuDNN/FA4 training integration environment", +) + + +def _relative_l2(actual, expected): + difference = reference = 0.0 + for a, b in zip(actual, expected, strict=True): + a, b = a.cuda().float(), b.cuda().float() + difference += (a - b).square().sum().item() + reference += b.square().sum().item() + return (difference / max(reference, 1e-30)) ** 0.5 + + +def run_mha64_training(context_length, *, hidden_size=128, vocab_size=1024, + num_anchors=16, steps=3, gradient_checkpointing=False): + """Compare identical-state VJPs and independent three-update trajectories.""" + from aspec.models.draft.dspark import DSparkConfig, DSparkDraftModel + from aspec.models.dspark import DSparkModel + from aspec.training.optimizer import BF16Optimizer + + class CausalBlockDSpark(DSparkModel): + # Exercise causal block visibility through the existing mask-policy hook. + def _block_mask_options(self): + return {"is_causal": True} + + torch.manual_seed(811) + config = DSparkConfig( + hidden_size=hidden_size, intermediate_size=3 * hidden_size, + num_hidden_layers=5, num_attention_heads=64, num_key_value_heads=64, + head_dim=64, vocab_size=vocab_size, mask_token_id=vocab_size - 1, + rms_norm_eps=1e-5, max_position_embeddings=1048576, rope_theta=10000, + num_target_layers=6, target_hidden_size=hidden_size, + target_num_hidden_layers=45, sliding_window=2048, + block_size=8, markov_rank=256, + ) + initial = DSparkDraftModel(config).bfloat16() + initial.freeze_embedding() + # Non-unit Q/K scales exercise the intermediate-rounding boundaries. + with torch.no_grad(): + scale = torch.linspace(0.7, 1.3, 64) + for layer in initial.layers: + layer.self_attn.q_norm.weight.copy_(scale) + layer.self_attn.k_norm.weight.copy_(scale.flip(0)) + state = {name: value.detach().cpu().clone() for name, value in initial.state_dict().items()} + del initial + inputs = torch.randint(vocab_size - 1, (1, context_length), device="cuda") + features = torch.randn(1, context_length, 6 * hidden_size, + device="cuda", dtype=torch.bfloat16) + teacher = torch.randn(1, context_length, hidden_size, device="cuda", dtype=torch.bfloat16) + head = torch.randn(vocab_size, hidden_size, device="cuda", dtype=torch.bfloat16) + head /= hidden_size ** 0.5 + supervision = torch.ones_like(inputs) + supervision[:, -1] = 0 + records, snapshots, reference_states = {}, {}, [] + for backend in ("fa4", "flex_attention", "cudnn_flex_attention"): + draft = DSparkDraftModel(config).bfloat16() + draft.load_state_dict(state) + if gradient_checkpointing: + draft.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + draft.freeze_embedding() + draft.cuda().train() + model = CausalBlockDSpark( + draft, block_size=8, num_anchors=num_anchors, + attention_backend=backend, deterministic_anchor_sampling=True, + ) + optimizer = BF16Optimizer( + draft, lr=6e-4, min_lr=6e-5, total_steps=100, warmup_ratio=0, + weight_decay=0, max_grad_norm=1.0, + ) + history, states = [], [] + for step in range(steps): + aligned_gradients = aligned_terms = aligned_loss = None + if backend == "fa4": + reference_states.append({name: value.detach().cpu().clone() + for name, value in draft.state_dict().items()}) + elif step: + # A backend precision comparison must use identical weights. + # Preserve the independent trajectory around this read-only VJP replay. + own_state = {name: value.detach().cpu().clone() + for name, value in draft.state_dict().items()} + draft.load_state_dict(reference_states[step]) + aligned = model(inputs, features, supervision, head, teacher, anchor_seed=911 + step) + aligned[0].backward() + aligned_gradients = [p.grad.detach().cpu().clone() + for p in draft.parameters() if p.requires_grad] + aligned_terms = {name: float(numerator.detach() / denominator.detach().clamp_min(1e-8)) + for name, (numerator, denominator) in aligned[5].items()} + aligned_loss = float(aligned[0].detach()) + optimizer.zero_grad() + draft.load_state_dict(own_state) + del aligned, own_state + result = model(inputs, features, supervision, head, teacher, anchor_seed=911 + step) + loss = result[0] + loss.backward() + parameters = [(name, p) for name, p in draft.named_parameters() if p.requires_grad] + assert all(p.grad is not None and torch.isfinite(p.grad).all() for _, p in parameters) + gradients = [p.grad.detach().cpu().clone() for _, p in parameters] + terms = {name: float(numerator.detach() / denominator.detach().clamp_min(1e-8)) + for name, (numerator, denominator) in result[5].items()} + if aligned_gradients is None: + aligned_gradients, aligned_terms, aligned_loss = gradients, terms, float(loss.detach()) + qk_backends = [layer.self_attn.qk_prepare_backend for layer in draft.layers] + norm = optimizer.step() + masters = [p.detach().cpu().clone() for p in optimizer.fp32_params] + assert all(torch.isfinite(p).all() for p in masters) + assert torch.equal(draft.embed_tokens.weight.cpu(), state["embed_tokens.weight"]) + history.append({"loss": float(loss.detach()), "components": terms, + "grad_norm": float(norm), "qk_backends": qk_backends, + "aligned_loss": aligned_loss, "aligned_components": aligned_terms}) + states.append((gradients, masters, aligned_gradients)) + records[backend], snapshots[backend] = history, states + del model, draft, optimizer, result, loss, parameters, gradients, masters + gc.collect() + torch.cuda.empty_cache() + metrics = [] + for step in range(steps): + comparisons = {} + for candidate, baseline in (("fa4", "flex_attention"), + ("cudnn_flex_attention", "fa4"), + ("cudnn_flex_attention", "flex_attention")): + a, b = records[candidate][step], records[baseline][step] + comparisons[candidate + "_vs_" + baseline] = { + "loss_absolute": abs(a["loss"] - b["loss"]), + "loss_relative": abs(a["loss"] - b["loss"]) / max(abs(b["loss"]), 1e-12), + "gradient_relative_l2": _relative_l2(snapshots[candidate][step][0], + snapshots[baseline][step][0]), + "aligned_gradient_relative_l2": _relative_l2(snapshots[candidate][step][2], + snapshots[baseline][step][2]), + "master_relative_l2": _relative_l2(snapshots[candidate][step][1], + snapshots[baseline][step][1]), + "component_absolute": {name: abs(a["components"][name] - b["components"][name]) + for name in a["components"]}, + "aligned_loss_relative": abs(a["aligned_loss"] - b["aligned_loss"]) + / max(abs(b["aligned_loss"]), 1e-12), + "aligned_component_absolute": { + name: abs(a["aligned_components"][name] - b["aligned_components"][name]) + for name in a["aligned_components"]}, + } + metrics.append(comparisons) + return {"context_length": context_length, + "hidden_size": hidden_size, "vocab_size": vocab_size, + "gradient_checkpointing": gradient_checkpointing, + "steps": steps, "records": records, "metrics": metrics} + + +@pytest.mark.parametrize("context_length", [8192, 262144]) +@pytest.mark.parametrize("full_geometry", [ + pytest.param(False, id="bounded"), + pytest.param(True, id="full", marks=pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUDNN_MHA64_FULL") != "1", + reason="requires explicit full-width training validation", + )), +]) +def test_mha64_training_loss_and_updates(context_length, full_geometry): + geometry = (dict(hidden_size=4096, vocab_size=154880, num_anchors=64, + gradient_checkpointing=context_length >= 262144) + if full_geometry else {}) + result = run_mha64_training(context_length, **geometry) + assert all(set(step["qk_backends"]) == {"aspec_cutedsl"} + for step in result["records"]["cudnn_flex_attention"]) + for step, comparisons in enumerate(result["metrics"]): + for pair, metric in comparisons.items(): + assert metric["loss_relative"] < 1e-3, result + assert metric["aligned_loss_relative"] < 1e-3, result + assert metric["aligned_gradient_relative_l2"] < 0.01, result + assert metric["master_relative_l2"] < 1e-3, result + baseline = result["records"][pair.split("_vs_")[1]][step] + for actual, expected in (("component_absolute", "components"), + ("aligned_component_absolute", "aligned_components")): + for name, error in metric[actual].items(): + assert error <= 1e-4 + 1e-3 * abs(baseline[expected][name]), result + + +@pytest.mark.parametrize("kv_heads,mode,error", [(16, "native", "Hq=Hkv"), + (64, "internal", "Hq/Hkv=4")]) +def test_mha64_rejects_unvalidated_gqa_modes(kv_heads, mode, error): + from aspec.models.ops.cudnn_flex_attention import CudnnFlexMask, cudnn_flex_attention + + query = torch.zeros(1, 8, 64, 64, device="cuda", dtype=torch.bfloat16) + key = torch.zeros(1, 8, kv_heads, 64, device="cuda", dtype=torch.bfloat16) + mask = CudnnFlexMask(torch.full((1, 1, 8), 8, device="cuda", dtype=torch.int32)) + with pytest.raises(ValueError, match=error): + cudnn_flex_attention(query, key, key, mask=mask, scale=64 ** -0.5, gqa_mode=mode) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("batch", [1, 2]) +def test_mha64_fused_qk_preserves_eager_rounding(dtype, batch): + from aspec.kernels.cutedsl.dspark_qk_norm_rope import compile_dspark_qk_norm_rope + from aspec.kernels.cutedsl.dspark_qk_norm_rope_bwd import compile_dspark_qk_norm_rope_bwd + from aspec.models.draft.dflash import _prepare_dflash_qk + from aspec.models.ops.dspark_qk_norm_rope import dspark_qk_norm_rope + + torch.manual_seed(93) + heads, dim, queries, context = 4, 64, 7, 33 + # Compile both modes for one geometry to verify cache isolation. + geometry = dict(dtype=dtype, h_q=heads, h_kv=heads, d=dim, rope_dim=dim, + eps=1e-5, batch_one=batch == 1) + compile_dspark_qk_norm_rope(**geometry, defer_secondary_loads=False) + compile_dspark_qk_norm_rope_bwd(**geometry) + shapes = ((batch, queries, heads * dim), (batch, context, heads * dim), + (batch, queries, heads * dim), (dim,), (dim,)) + data = [torch.randn(shape, device="cuda", dtype=dtype) for shape in shapes] + data[3].copy_(torch.linspace(0.3, 1.7, dim, device="cuda")) + data[4].copy_(data[3].flip(0)) + angles = torch.randn(257, dim // 2, device="cuda") + angles = torch.cat((angles, angles), dim=-1) + cos, sin = angles.cos()[None, None].to(dtype), angles.sin()[None, None].to(dtype) + positions = [torch.randint(257, (batch, n), device="cuda") for n in (queries, context)] + gradients = [torch.randn(batch, n, heads, dim, device="cuda", dtype=dtype) + for n in (queries, context + queries)] + values = [] + for fused in (False, True): + q, k, kd, wq, wk = [t.clone().requires_grad_() for t in data] + if fused: + output = dspark_qk_norm_rope(q, k, kd, wq, wk, 1e-5, cos, sin, + positions[0], positions[1], positions[0], heads, heads, dim) + else: + output = _prepare_dflash_qk(q, k, kd, wq, wk, 1e-5, cos, sin, + positions[0], torch.cat((positions[1], positions[0]), dim=1), + heads, heads, dim) + grads = torch.autograd.grad(output, (q, k, kd, wq, wk), gradients) + values.append((output, grads)) + for actual, expected in zip(values[1][0], values[0][0], strict=True): + assert _relative_l2([actual], [expected]) < 1e-4 + for actual, expected in zip(values[1][1], values[0][1], strict=True): + assert _relative_l2([actual], [expected]) < 1e-3 From 366d28dc1cdb2620d90b7923dd6f12b9a9c31188 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Wed, 18 Nov 2026 12:36:08 +0800 Subject: [PATCH 2/4] fix(attention): preserve established Q/K preparation semantics --- aspec/kernels/cutedsl/dspark_qk_norm_rope.py | 28 +----- .../cutedsl/dspark_qk_norm_rope_bwd.py | 48 ++--------- aspec/models/ops/dspark_qk_norm_rope.py | 2 - docs/user-guide/structures.md | 4 +- tests/test_cudnn_mha64.py | 85 ++++++++++--------- 5 files changed, 57 insertions(+), 110 deletions(-) diff --git a/aspec/kernels/cutedsl/dspark_qk_norm_rope.py b/aspec/kernels/cutedsl/dspark_qk_norm_rope.py index 1d27c62..9285279 100644 --- a/aspec/kernels/cutedsl/dspark_qk_norm_rope.py +++ b/aspec/kernels/cutedsl/dspark_qk_norm_rope.py @@ -2,15 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 # # Adapted by ASpec Authors from NVIDIA cuDNN Frontend cb8cec8. -# ASpec modifications Copyright (c) 2026 Ant Group Co., Ltd. """ASpec-owned CuTeDSL kernel for DSpark Q/K RMSNorm and indexed RoPE. The kernel reads draft Q plus separate context/draft K planes and writes compact Q and logical ``[context, draft]`` K outputs without materializing a K concat. -The legacy path rounds once after FP32 normalization and rotation. The optional -intermediate-rounding mode preserves the eager input-dtype arithmetic boundaries. -Normalization reductions remain FP32 in both modes. +RMSNorm and RoPE arithmetic stay in FP32 and round once to the input dtype. """ from typing import NamedTuple, Optional @@ -127,7 +124,6 @@ def frost_qk_norm_rope( const_head_counts: cutlass.Constexpr[bool], batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], - round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """``rows_per_group`` consecutive rows per lane group: ALL loads issued first, then the reductions and stores. @@ -359,13 +355,7 @@ def frost_qk_norm_rope( ys = [] for c in cutlass.range_constexpr(chunks): if cutlass.const_expr(apply_norm): - if cutlass.const_expr(round_intermediates): - normalized = [(xs[r][c][i] * rstd).to(mQ.element_type).to(cutlass.Float32) - for i in range(ELEMS_PER_ACCESS)] - ys.append([(normalized[i] * ws[r][c][i]).to(mQ.element_type).to(cutlass.Float32) - for i in range(ELEMS_PER_ACCESS)]) - else: - ys.append([xs[r][c][i] * rstd * ws[r][c][i] for i in range(ELEMS_PER_ACCESS)]) + ys.append([xs[r][c][i] * rstd * ws[r][c][i] for i in range(ELEMS_PER_ACCESS)]) else: ys.append(list(xs[r][c])) @@ -378,12 +368,7 @@ def frost_qk_norm_rope( for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): partner = cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, ys[0][i], cutlass.Int32(half_lanes), 31, kind=nvvm.Shfl.BFLY)) signed = -partner if lane < cutlass.Int32(half_lanes) else partner - if cutlass.const_expr(round_intermediates): - direct = (ys[0][i] * cs[r][i]).to(mQ.element_type).to(cutlass.Float32) - other = (signed * ss[r][i]).to(mQ.element_type).to(cutlass.Float32) - rotated.append(direct + other) - else: - rotated.append(cute.math.fma(ys[0][i], cs[r][i], signed * ss[r][i])) + rotated.append(cute.math.fma(ys[0][i], cs[r][i], signed * ss[r][i])) for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): ys[0][i] = rotated[i] if in_rope else ys[0][i] @@ -472,7 +457,6 @@ def qk_norm_rope_launch( batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], stream: cuda.CUstream, - round_intermediates: cutlass.Constexpr[bool] = False, ): frost_qk_norm_rope( q, @@ -506,7 +490,6 @@ def qk_norm_rope_launch( const_head_counts, batch_one, use_pdl, - round_intermediates, ).launch(grid=(n_blocks, 1, 1), block=(threads_per_cta, 1, 1), stream=stream, use_pdl=use_pdl) @@ -529,7 +512,6 @@ class DSparkQkNormRopeRecipe(NamedTuple): eps: float rows_per_cta: int batch_one: bool = False - round_intermediates: bool = False def _fake(dtype, shape, stride_order): @@ -556,7 +538,6 @@ def compile_dspark_qk_norm_rope( const_head_counts: bool = DEFAULT_CONST_HEAD_COUNTS, batch_one: bool = False, use_pdl: bool = False, - round_intermediates: bool = False, ) -> DSparkQkNormRopeRecipe: """Compile the DSpark asymmetric, indexed-RoPE training specialization.""" @@ -576,7 +557,6 @@ def compile_dspark_qk_norm_rope( bool(const_head_counts), bool(batch_one), bool(use_pdl), - bool(round_intermediates), torch.cuda.current_device(), ) if key not in compiled_cache: @@ -626,7 +606,6 @@ def compile_dspark_qk_norm_rope( bool(batch_one), bool(use_pdl), _FAKE_STREAM, - bool(round_intermediates), options="--enable-tvm-ffi", ) return DSparkQkNormRopeRecipe( @@ -638,7 +617,6 @@ def compile_dspark_qk_norm_rope( eps=float(eps), rows_per_cta=(threads_per_cta // lanes_per_row(d)) * rows_per_group, batch_one=bool(batch_one), - round_intermediates=bool(round_intermediates), ) diff --git a/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py b/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py index 42e5472..b3f92ab 100644 --- a/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py +++ b/aspec/kernels/cutedsl/dspark_qk_norm_rope_bwd.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # # Adapted by ASpec Authors from NVIDIA cuDNN Frontend cb8cec8. -# ASpec modifications Copyright (c) 2026 Ant Group Co., Ltd. """ASpec-owned CuTeDSL adjoint for the DSpark Q/K preparation kernel. @@ -132,7 +131,6 @@ def _row_class( mAmaxOut: Optional[cute.Tensor], # [>= grid] fp32: partials[cta] = max |stored words| of this CTA (plain store at CTA end); None = no fold cta: cutlass.Int32, # the GRID-level block index: the partial this CTA owns sred, # the ONE sRed SMEM Array of the kernel, fp32 [warps] (None without want_amax) - round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """The persistent loop of ONE row class (Q, K or V) for one CTA. @@ -290,15 +288,9 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one rotated = [] for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): t_i = g[0][i] * sin_v[i] - if cutlass.const_expr(round_intermediates): - t_i = t_i.to(io_dtype).to(cutlass.Float32) partner = cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, t_i, cutlass.Int32(half_lanes), 31, kind=nvvm.Shfl.BFLY)) signed = partner if lane < cutlass.Int32(half_lanes) else -partner # the forward's sign, FLIPPED - if cutlass.const_expr(round_intermediates): - direct = (g[0][i] * cos_v[i]).to(io_dtype).to(cutlass.Float32) - rotated.append((direct + signed).to(io_dtype).to(cutlass.Float32)) - else: - rotated.append(cute.math.fma(g[0][i], cos_v[i], signed)) + rotated.append(cute.math.fma(g[0][i], cos_v[i], signed)) for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): g[0][i] = rotated[i] if in_rope else g[0][i] @@ -312,9 +304,6 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one rstd = rstds[r] x_hat = [[xs[r][c][i] * rstd for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] gw = [[g[c][i] * ws[c][i] for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] - if cutlass.const_expr(round_intermediates): - gw = [[gw[c][i].to(io_dtype).to(cutlass.Float32) - for i in range(ELEMS_PER_ACCESS)] for c in range(chunks)] acc = zero for c in cutlass.range_constexpr(chunks): for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): @@ -328,23 +317,15 @@ class arms specialise on ``h_ct`` / ``apply_norm`` / ``is_copy`` with one keep = valid & (~deads[r]) if cutlass.const_expr(has_seq_lens) else valid for c in cutlass.range_constexpr(chunks): for i in cutlass.range_constexpr(ELEMS_PER_ACCESS): - if cutlass.const_expr(round_intermediates): - normalized = x_hat[c][i].to(io_dtype).to(cutlass.Float32) - contribution = (g[c][i] * normalized).to(io_dtype).to(cutlass.Float32) - dw_acc[c * ELEMS_PER_ACCESS + i] = ( - dw_acc[c * ELEMS_PER_ACCESS + i] + contribution - if keep else dw_acc[c * ELEMS_PER_ACCESS + i] - ) - else: - dw_acc[c * ELEMS_PER_ACCESS + i] = ( - cute.math.fma( - g[c][i], - x_hat[c][i], - dw_acc[c * ELEMS_PER_ACCESS + i], - ) - if keep - else dw_acc[c * ELEMS_PER_ACCESS + i] + dw_acc[c * ELEMS_PER_ACCESS + i] = ( + cute.math.fma( + g[c][i], + x_hat[c][i], + dw_acc[c * ELEMS_PER_ACCESS + i], ) + if keep + else dw_acc[c * ELEMS_PER_ACCESS + i] + ) else: # RoPE-only: dx IS g; [rope_dim, D) goes back through fp32_to_fp16 # untouched by any fp32 op -> a bit-exact copy (asserted by the tests). @@ -456,7 +437,6 @@ def frost_qk_norm_rope_bwd( rows_per_group: cutlass.Constexpr[int], batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], - round_intermediates: cutlass.Constexpr[bool] = False, ) -> None: """Three CTA classes over one grid (Q | K | V), each a persistent row loop; see the module docstring.""" if cutlass.const_expr(use_pdl): @@ -544,7 +524,6 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, - round_intermediates, ) else: if cta < n_ctas_q + n_ctas_k: @@ -606,7 +585,6 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, - round_intermediates, ) else: # ---- V class: the band copy (no norm, no RoPE, no dW) -------------------- @@ -660,7 +638,6 @@ def frost_qk_norm_rope_bwd( mAmaxOut, cta, sred, - round_intermediates, ) if cutlass.const_expr(use_pdl): @@ -794,7 +771,6 @@ def qk_norm_rope_bwd_launch( batch_one: cutlass.Constexpr[bool], use_pdl: cutlass.Constexpr[bool], stream: cuda.CUstream, - round_intermediates: cutlass.Constexpr[bool] = False, ): frost_qk_norm_rope_bwd( dq, @@ -839,7 +815,6 @@ def qk_norm_rope_bwd_launch( rows_per_group, batch_one, use_pdl, - round_intermediates, ).launch(grid=(n_blocks, 1, 1), block=(threads_per_cta, 1, 1), stream=stream, use_pdl=use_pdl) @@ -899,7 +874,6 @@ class QkNormRopeBwdRecipe(NamedTuple): # Appended (default = today's artifact): the amax fold of the stored bands as per-CTA partials (``amax_out`` REQUIRED). want_amax: bool = False batch_one: bool = False - round_intermediates: bool = False def _resolve_n_ctas_cap(policy: str, device: int) -> int: @@ -931,7 +905,6 @@ def compile_dspark_qk_norm_rope_bwd( batch_one: bool = False, use_pdl: bool = False, n_ctas_policy: str = DEFAULT_N_CTAS_POLICY, - round_intermediates: bool = False, ) -> QkNormRopeBwdRecipe: """Compile the DSpark asymmetric, indexed-RoPE training adjoint. @@ -966,7 +939,6 @@ def compile_dspark_qk_norm_rope_bwd( bool(const_head_counts), bool(batch_one), bool(use_pdl), - bool(round_intermediates), str(n_ctas_policy), device, ) @@ -1035,7 +1007,6 @@ def compile_dspark_qk_norm_rope_bwd( bool(batch_one), bool(use_pdl), _FAKE_STREAM, - bool(round_intermediates), options="--enable-tvm-ffi", ) reduce_key = (int(d), device) @@ -1073,7 +1044,6 @@ def compile_dspark_qk_norm_rope_bwd( rows_per_group=int(rows_per_group), want_amax=False, batch_one=bool(batch_one), - round_intermediates=bool(round_intermediates), ) diff --git a/aspec/models/ops/dspark_qk_norm_rope.py b/aspec/models/ops/dspark_qk_norm_rope.py index 3ee19a3..d6cfefa 100644 --- a/aspec/models/ops/dspark_qk_norm_rope.py +++ b/aspec/models/ops/dspark_qk_norm_rope.py @@ -92,7 +92,6 @@ def _forward_recipe( eps=eps, defer_secondary_loads=False, batch_one=batch_one, - round_intermediates=d == 64 and h_q == h_kv, ) _FWD_RECIPES[key] = recipe return recipe @@ -119,7 +118,6 @@ def _backward_recipe( rope_dim=d, eps=eps, batch_one=batch_one, - round_intermediates=d == 64 and h_q == h_kv, ) _BWD_RECIPES[key] = recipe return recipe diff --git a/docs/user-guide/structures.md b/docs/user-guide/structures.md index 8c75e4b..3f35071 100644 --- a/docs/user-guide/structures.md +++ b/docs/user-guide/structures.md @@ -36,9 +36,7 @@ The adapter also supports width-128 GQA and materialized MLA with query/key widt The optional internal joint-GQA backward has a separate contract: head width 128 and four query heads per KV head. Native MHA64 does not use that specialization. -Its fused Q/K normalization and RoPE preserve the eager FP16/BF16 intermediate -rounding boundaries while retaining FP32 normalization reductions. Keep the -attention backend fixed when reproducing or resuming a training run: identical-state +Keep the attention backend fixed when reproducing or resuming a training run: identical-state loss/gradient comparisons and independent optimizer trajectories are different checks. Run the MHA64 and retained attention/kernel regressions in the Linux Draft environment: diff --git a/tests/test_cudnn_mha64.py b/tests/test_cudnn_mha64.py index 1362596..2ccccca 100644 --- a/tests/test_cudnn_mha64.py +++ b/tests/test_cudnn_mha64.py @@ -47,7 +47,7 @@ def _block_mask_options(self): ) initial = DSparkDraftModel(config).bfloat16() initial.freeze_embedding() - # Non-unit Q/K scales exercise the intermediate-rounding boundaries. + # Non-unit Q/K scales exercise learned normalization weights. with torch.no_grad(): scale = torch.linspace(0.7, 1.3, 64) for layer in initial.layers: @@ -196,44 +196,47 @@ def test_mha64_rejects_unvalidated_gqa_modes(kv_heads, mode, error): @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) -@pytest.mark.parametrize("batch", [1, 2]) -def test_mha64_fused_qk_preserves_eager_rounding(dtype, batch): - from aspec.kernels.cutedsl.dspark_qk_norm_rope import compile_dspark_qk_norm_rope - from aspec.kernels.cutedsl.dspark_qk_norm_rope_bwd import compile_dspark_qk_norm_rope_bwd - from aspec.models.draft.dflash import _prepare_dflash_qk - from aspec.models.ops.dspark_qk_norm_rope import dspark_qk_norm_rope - - torch.manual_seed(93) - heads, dim, queries, context = 4, 64, 7, 33 - # Compile both modes for one geometry to verify cache isolation. - geometry = dict(dtype=dtype, h_q=heads, h_kv=heads, d=dim, rope_dim=dim, - eps=1e-5, batch_one=batch == 1) - compile_dspark_qk_norm_rope(**geometry, defer_secondary_loads=False) - compile_dspark_qk_norm_rope_bwd(**geometry) - shapes = ((batch, queries, heads * dim), (batch, context, heads * dim), - (batch, queries, heads * dim), (dim,), (dim,)) - data = [torch.randn(shape, device="cuda", dtype=dtype) for shape in shapes] - data[3].copy_(torch.linspace(0.3, 1.7, dim, device="cuda")) - data[4].copy_(data[3].flip(0)) - angles = torch.randn(257, dim // 2, device="cuda") - angles = torch.cat((angles, angles), dim=-1) - cos, sin = angles.cos()[None, None].to(dtype), angles.sin()[None, None].to(dtype) - positions = [torch.randint(257, (batch, n), device="cuda") for n in (queries, context)] - gradients = [torch.randn(batch, n, heads, dim, device="cuda", dtype=dtype) - for n in (queries, context + queries)] - values = [] - for fused in (False, True): - q, k, kd, wq, wk = [t.clone().requires_grad_() for t in data] - if fused: - output = dspark_qk_norm_rope(q, k, kd, wq, wk, 1e-5, cos, sin, - positions[0], positions[1], positions[0], heads, heads, dim) +def test_mha64_native_attention_matches_flex(dtype): + from aspec.models.dflash import _create_dflash_mask_mod + from aspec.models.ops.cudnn_flex_attention import build_dspark_mla_mask, cudnn_flex_attention + from aspec.models.ops.flex_attention import ( + compile_friendly_create_block_mask, + compile_friendly_flex_attention, + ) + + generator = torch.Generator(device="cuda").manual_seed(93) + context_length, block_size, heads, dim = 71, 8, 4, 64 + anchors = torch.tensor([[17, 63]], device="cuda") + keep = torch.ones_like(anchors, dtype=torch.bool) + queries = block_size * anchors.shape[1] + mask_options = dict(is_causal=True, context_window=32) + mask_mod = _create_dflash_mask_mod(anchors, keep, context_length, block_size, **mask_options) + flex_mask = compile_friendly_create_block_mask( + mask_mod, B=1, H=None, Q_LEN=queries, KV_LEN=context_length + queries, + device=torch.device("cuda"), + ) + cudnn_mask = build_dspark_mla_mask( + anchor_positions=anchors, block_keep_mask=keep, context_length=context_length, + block_size=block_size, **mask_options, + ) + inputs = [torch.randn(1, n, heads, dim, device="cuda", dtype=dtype, generator=generator) + for n in (queries, context_length + queries, context_length + queries)] + dout = torch.randn(inputs[0].shape, device="cuda", dtype=dtype, generator=generator) + results = [] + for native in (False, True): + q, k, v = [x.detach().clone().requires_grad_() for x in inputs] + if native: + out = cudnn_flex_attention(q, k, v, mask=cudnn_mask, scale=dim ** -0.5, + gqa_mode="native") else: - output = _prepare_dflash_qk(q, k, kd, wq, wk, 1e-5, cos, sin, - positions[0], torch.cat((positions[1], positions[0]), dim=1), - heads, heads, dim) - grads = torch.autograd.grad(output, (q, k, kd, wq, wk), gradients) - values.append((output, grads)) - for actual, expected in zip(values[1][0], values[0][0], strict=True): - assert _relative_l2([actual], [expected]) < 1e-4 - for actual, expected in zip(values[1][1], values[0][1], strict=True): - assert _relative_l2([actual], [expected]) < 1e-3 + out = compile_friendly_flex_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), + block_mask=flex_mask, scale=dim ** -0.5, + ).transpose(1, 2) + gradients = torch.autograd.grad(out, (q, k, v), dout) + assert torch.isfinite(out).all() + assert all(torch.isfinite(g).all() for g in gradients) + results.append((out, gradients)) + assert _relative_l2([results[1][0]], [results[0][0]]) < 0.01 + for actual, expected in zip(results[1][1], results[0][1], strict=True): + assert _relative_l2([actual], [expected]) < 0.01 From b99257670102344f2713696958f5e8be7c955cb3 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Wed, 18 Nov 2026 13:33:47 +0800 Subject: [PATCH 3/4] docs(attention): clarify backend trajectory precision --- docs/user-guide/structures.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/structures.md b/docs/user-guide/structures.md index 3f35071..d3c5523 100644 --- a/docs/user-guide/structures.md +++ b/docs/user-guide/structures.md @@ -36,8 +36,9 @@ The adapter also supports width-128 GQA and materialized MLA with query/key widt The optional internal joint-GQA backward has a separate contract: head width 128 and four query heads per KV head. Native MHA64 does not use that specialization. -Keep the attention backend fixed when reproducing or resuming a training run: identical-state -loss/gradient comparisons and independent optimizer trajectories are different checks. +Compare loss and gradients at identical weights when validating backend precision. +Independent BF16 optimizer trajectories also include accumulated arithmetic and gradient-reduction +differences; keep the attention backend fixed when reproducing or resuming a training run. Run the MHA64 and retained attention/kernel regressions in the Linux Draft environment: @@ -50,8 +51,12 @@ ASPEC_RUN_CUDNN_FLEX_INTEGRATION=1 ASPEC_RUN_CUTEDSL_INTEGRATION=1 \ The MHA64 training tests use synthetic features and bounded hidden/vocabulary sizes. Add `ASPEC_RUN_CUDNN_MHA64_FULL=1` to include hidden size 4096, vocabulary size 154880 and 64 anchors at both 8K and 256K. The full-width 256K case enables gradient checkpointing -uniformly across FA4, FlexAttention and cuDNN. These numerical checks do not establish -real-checkpoint training quality or convergence. +uniformly across FA4, FlexAttention and cuDNN. The independent-trajectory checks retain a 0.1% +total-loss criterion and component tolerances of `atol=1e-4, rtol=1e-3`. In the synthetic 256K +control, FA4 versus FlexAttention alone shows about 0.105% third-update total-loss drift, with +confidence-loss drift also crossing its component tolerance. Fixed-weight and weight-swap controls +show that drift follows trained weight states while identical-state loss and gradients agree across +backends. These numerical checks do not establish real-checkpoint training quality or convergence. ## Packing and context parallelism From 443e079fa6973c429b46e7d3be2593730dadad66 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Wed, 18 Nov 2026 13:54:04 +0800 Subject: [PATCH 4/4] docs: preserve structures guide scope --- docs/user-guide/structures.md | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/docs/user-guide/structures.md b/docs/user-guide/structures.md index d3c5523..704c415 100644 --- a/docs/user-guide/structures.md +++ b/docs/user-guide/structures.md @@ -26,38 +26,6 @@ optimizer/FP32 master, gradients, activations and transfer buffers when checking resume/export preserves expert count, top-k, normalization, auxiliary policy and control state; missing or unverifiable routing state is not silently accepted. -## cuDNN attention geometry - -Set `training.attention_backend=cudnn_flex_attention` to select the cuDNN Frontend -backend in the [pinned runtime](installation.md). On SM90 and SM100, native MHA -supports query/key and value head widths of 64 with equal query and KV head counts. -The adapter also supports width-128 GQA and materialized MLA with query/key width -192 and value width 128. Width-64 GQA is outside the supported contract. - -The optional internal joint-GQA backward has a separate contract: head width 128 -and four query heads per KV head. Native MHA64 does not use that specialization. -Compare loss and gradients at identical weights when validating backend precision. -Independent BF16 optimizer trajectories also include accumulated arithmetic and gradient-reduction -differences; keep the attention backend fixed when reproducing or resuming a training run. - -Run the MHA64 and retained attention/kernel regressions in the Linux Draft environment: - -```bash -ASPEC_RUN_CUDNN_FLEX_INTEGRATION=1 ASPEC_RUN_CUTEDSL_INTEGRATION=1 \ - bash tools/run_in_env.sh python -m pytest -q \ - tests/test_cudnn_mha64.py tests/test_cudnn_flex_attention.py tests/test_dspark_qk_cutedsl.py -``` - -The MHA64 training tests use synthetic features and bounded hidden/vocabulary sizes. -Add `ASPEC_RUN_CUDNN_MHA64_FULL=1` to include hidden size 4096, vocabulary size 154880 -and 64 anchors at both 8K and 256K. The full-width 256K case enables gradient checkpointing -uniformly across FA4, FlexAttention and cuDNN. The independent-trajectory checks retain a 0.1% -total-loss criterion and component tolerances of `atol=1e-4, rtol=1e-3`. In the synthetic 256K -control, FA4 versus FlexAttention alone shows about 0.105% third-update total-loss drift, with -confidence-loss drift also crossing its component tolerance. Fixed-weight and weight-swap controls -show that drift follows trained weight states while identical-state loss and gradients agree across -backends. These numerical checks do not establish real-checkpoint training quality or convergence. - ## Packing and context parallelism ASpec supports two dense attention layouts. The padded-document layout keeps one