diff --git a/aspec/models/ops/absorbed_mla.py b/aspec/models/ops/absorbed_mla.py new file mode 100644 index 0000000..b7cd70b --- /dev/null +++ b/aspec/models/ops/absorbed_mla.py @@ -0,0 +1,157 @@ +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reference absorbed-MLA attention for feasibility checks. + +The standard DeepSeek-V2/V3 identity re-associates the materialized MLA +projections: + + q_nope @ (c_kv @ W_UK^T)^T == (q_nope @ W_UK) @ c_kv^T + sum_t p_t (c_kv_t @ W_UV^T) == (sum_t p_t c_kv_t) @ W_UV^T + +so the attention boundary can consume the normalized latent ``c_kv`` directly +instead of materializing per-head ``k_nope`` and ``v``. RoPE is still applied to +the small ``qk_rope_head_dim`` side, and the two score terms are added before +the softmax. + +This module is a dense-mask reference implementation used by numerical and +feasibility checks. It is not wired into training, and it does not implement the +FlexAttention, cuDNN THD, Direct-CP or fused-kernel paths needed for a +production claim. Dense boolean masks are broadcast against ``[B, H, S, T]``. +""" + +from __future__ import annotations + +import math + +import torch + + +def split_kv_b_proj( + weight: torch.Tensor, + *, + num_heads: int, + qk_nope_head_dim: int, + v_head_dim: int, + kv_lora_rank: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Split ``kv_b_proj.weight`` into per-head ``W_UK`` and ``W_UV`` views.""" + expected = (num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank) + if tuple(weight.shape) != expected: + raise ValueError( + "kv_b_proj.weight must have shape " + f"{expected}, got {tuple(weight.shape)}" + ) + view = weight.view(num_heads, qk_nope_head_dim + v_head_dim, kv_lora_rank) + return view[:, :qk_nope_head_dim, :], view[:, qk_nope_head_dim:, :] + + +def absorbed_mla_attention( + q_nope: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + kv_b_proj_weight: torch.Tensor, + *, + softmax_scale: float | None = None, + attn_mask: torch.Tensor | None = None, +) -> torch.Tensor: + """Absorbed MLA attention with a dense mask. + + Args: + q_nope: ``[B, S, H, Dn]`` query without RoPE. + q_rope: ``[B, S, H, Dr]`` query after RoPE. + compressed_kv: ``[B, T, K]`` normalized latent KV before ``kv_b_proj``. + k_rope: ``[B, T, Dr]`` shared MQA key after RoPE. + kv_b_proj_weight: ``[H * (Dn + Dv), K]`` materialized projection weight. + softmax_scale: score scale; defaults to ``1 / sqrt(Dn + Dr)``. + attn_mask: optional boolean mask broadcastable to ``[B, H, S, T]``. + + Returns: + ``[B, S, H * Dv]`` attention output before ``o_proj``. + """ + if q_nope.ndim != 4 or q_rope.ndim != 4: + raise ValueError("q_nope and q_rope must be [B, S, H, D]") + batch, query_len, num_heads, qk_nope_head_dim = q_nope.shape + if q_rope.shape[:3] != q_nope.shape[:3]: + raise ValueError("q_rope must match q_nope's batch, query and head axes") + qk_rope_head_dim = q_rope.size(-1) + if compressed_kv.ndim != 3: + raise ValueError("compressed_kv must be [B, T, K]") + if compressed_kv.shape[0] != batch: + raise ValueError("compressed_kv batch size must match the queries") + key_len, kv_lora_rank = compressed_kv.shape[1], compressed_kv.shape[2] + if k_rope.shape != (batch, key_len, qk_rope_head_dim): + raise ValueError( + "k_rope must be [B, T, qk_rope_head_dim] with the compressed_kv " + "batch and key length" + ) + if kv_b_proj_weight.ndim != 2: + raise ValueError("kv_b_proj.weight must be a matrix") + inferred_v_head_dim = ( + kv_b_proj_weight.shape[0] // num_heads - qk_nope_head_dim + ) + if inferred_v_head_dim <= 0 or kv_b_proj_weight.shape[0] % num_heads: + raise ValueError("kv_b_proj.weight rows do not describe per-head K/V") + w_uk, w_uv = split_kv_b_proj( + kv_b_proj_weight, + num_heads=num_heads, + qk_nope_head_dim=qk_nope_head_dim, + v_head_dim=inferred_v_head_dim, + kv_lora_rank=kv_lora_rank, + ) + + dtype = q_nope.dtype + for name, tensor in ( + ("q_rope", q_rope), + ("compressed_kv", compressed_kv), + ("k_rope", k_rope), + ("kv_b_proj.weight", kv_b_proj_weight), + ): + if tensor.dtype != dtype: + raise ValueError(f"{name} dtype must match q_nope ({dtype})") + if not dtype.is_floating_point: + raise ValueError("absorbed MLA requires floating-point tensors") + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim) + if ( + isinstance(softmax_scale, bool) + or not math.isfinite(softmax_scale) + or softmax_scale <= 0 + ): + raise ValueError("softmax_scale must be positive and finite") + + q_latent = torch.einsum("bshn,hnk->bshk", q_nope, w_uk) + scores = torch.einsum("bshk,btk->bhst", q_latent, compressed_kv) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope, k_rope) + scores = scores * softmax_scale + if attn_mask is not None: + if attn_mask.dtype != torch.bool: + raise ValueError("attn_mask must be boolean") + try: + mask = attn_mask.expand(scores.shape) + except RuntimeError as exc: + raise ValueError( + "attn_mask must broadcast against [B, H, S, T]" + ) from exc + scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores.float(), dim=-1) + if attn_mask is not None: + # Match fused SDPA: rows with no visible key contribute zero, not a + # uniform average over the finite masked scores. + probs = probs * mask.any(dim=-1, keepdim=True) + probs = probs.to(dtype) + out_latent = torch.einsum("bhst,btk->bshk", probs, compressed_kv) + output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) + return output.reshape(batch, query_len, num_heads * inferred_v_head_dim) diff --git a/aspec/models/ops/absorbed_mla_fused.py b/aspec/models/ops/absorbed_mla_fused.py new file mode 100644 index 0000000..2d063f2 --- /dev/null +++ b/aspec/models/ops/absorbed_mla_fused.py @@ -0,0 +1,796 @@ +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fused absorbed-MLA attention with a FlashAttention-style Triton kernel. + +The reference implementation in :mod:`aspec.models.ops.absorbed_mla` keeps the +logits dense and materializes them with ``einsum``. This module keeps the same +absorbed identity but fuses the score, online softmax and latent-output +accumulation, so it never materializes ``[B, H, S, T]`` scores or per-head +``k_nope``/``v`` tensors: + + q_lat = q_nope @ W_UK # [B, S, H, K] + logits = q_lat @ c_kv^T + q_rope @ k_rope^T + out = softmax(logits) @ c_kv # [B, S, H, K] + output = out @ W_UV^T # [B, S, H * Dv] + +``W_UK``/``W_UV`` are per-head blocks of ``kv_b_proj.weight``; the latent +projection before and after the attention core stay in PyTorch so the gradient +for ``q_nope``/``kv_b_proj.weight`` flows through normal autograd. + +This is a feasibility kernel, not a production path: it supports dense, +unmasked attention on contiguous BSHD/BTD tensors, and dense masks remain on the +reference implementation. Backward uses two deterministic kernels (query-side +and key/value-side) and never needs atomics. +""" + +from __future__ import annotations + +import math + +import torch + +from aspec.models.ops.absorbed_mla import split_kv_b_proj + +try: # pragma: no cover - import guard for CPU-only installs + import triton + import triton.language as tl + + _HAS_TRITON = True +except ImportError: # pragma: no cover + triton = None + tl = None + _HAS_TRITON = False + + +def _next_power_of_2(value: int) -> int: + return 1 << max(0, int(value) - 1).bit_length() + + +# The latent output forces a [BLOCK_N, BLOCK_K] FP32 accumulator in the +# key/value-side backward, so the key tile stays smaller than the query tile. +# ``_DKDV_GROUP`` bounds the head-reduction workspace: each program owns +# ``num_heads / GROUP`` partial latent-gradient tensors instead of one per head. +_FWD_TILE_M = 32 +_FWD_TILE_N = 64 +_FWD_WARPS = 4 +_FWD_STAGES = 2 +_DQ_TILE_M = 16 +_DQ_TILE_N = 64 +_DQ_WARPS = 4 +_DQ_STAGES = 2 +_DKDV_TILE_M = 64 +_DKDV_TILE_N = 32 +_DKDV_WARPS = 4 +_DKDV_STAGES = 2 +_DKDV_GROUP = 8 + + +def _tile_sizes( + key_dim: int, + rope_dim: int, + block_m: int, + block_n: int, + query_len: int, +) -> tuple[int, int, int, int]: + block_k = max(16, _next_power_of_2(key_dim)) + block_dr = max(16, _next_power_of_2(rope_dim)) + # A long configured query tile is wasted on short draft blocks. + block_m = max(16, min(block_m, _next_power_of_2(query_len))) + return block_m, block_n, block_k, block_dr + + +def _dkdv_group(num_heads: int, preferred: int | None = None) -> int: + if preferred is None: + preferred = _DKDV_GROUP + group = max(1, min(int(preferred), int(num_heads))) + while num_heads % group: + group -= 1 + return group + + +if _HAS_TRITON: + + @triton.jit + def _absorbed_mla_fwd_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + out_ptr, + lse_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_o_b, + stride_o_s, + stride_o_h, + stride_l_b, + stride_l_h, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + ): + pid_m = tl.program_id(0) + pid_bh = tl.program_id(1) + b = pid_bh // H + h = pid_bh % H + + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_m = offs_m < S + mask_k = offs_k < K + mask_dr = offs_dr < DR + + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + + m_i = tl.full([BLOCK_M], float("-inf"), tl.float32) + l_i = tl.zeros([BLOCK_M], tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_K], tl.float32) + + for start_n in range(0, T, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + mask_n = offs_n < T + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + + m_new = tl.maximum(m_i, tl.max(scores, 1)) + alpha = tl.exp(m_i - m_new) + probs = tl.exp(scores - m_new[:, None]) + l_i = l_i * alpha + tl.sum(probs, 1) + acc = acc * alpha[:, None] + tl.dot(probs.to(c.dtype), c) + m_i = m_new + + l_safe = tl.where(l_i > 0, l_i, 1.0) + acc = acc / l_safe[:, None] + lse = m_i + tl.log(l_safe) + tl.store( + out_ptr + + b * stride_o_b + + offs_m[:, None] * stride_o_s + + h * stride_o_h + + offs_k[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_k[None, :], + ) + tl.store(lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, lse, mask=mask_m) + + @triton.jit + def _absorbed_mla_bwd_dq_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + dout_ptr, + lse_ptr, + d_ptr, + dq_lat_ptr, + dq_rope_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_do_b, + stride_do_s, + stride_do_h, + stride_l_b, + stride_l_h, + stride_d_b, + stride_d_s, + stride_d_h, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + ): + pid_m = tl.program_id(0) + pid_bh = tl.program_id(1) + b = pid_bh // H + h = pid_bh % H + + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_m = offs_m < S + mask_k = offs_k < K + mask_dr = offs_dr < DR + + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + dout = tl.load( + dout_ptr + + b * stride_do_b + + offs_m[:, None] * stride_do_s + + h * stride_do_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + lse = tl.load( + lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, + mask=mask_m, + other=0.0, + ) + d_i = tl.load( + d_ptr + b * stride_d_b + offs_m * stride_d_s + h * stride_d_h, + mask=mask_m, + other=0.0, + ) + + dq_lat = tl.zeros([BLOCK_M, BLOCK_K], tl.float32) + dq_rope = tl.zeros([BLOCK_M, BLOCK_DR], tl.float32) + + for start_n in range(0, T, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + mask_n = offs_n < T + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + probs = tl.exp(scores - lse[:, None]) + probs = tl.where(mask_m[:, None] & mask_n[None, :], probs, 0.0) + + dp = tl.dot(dout, tl.trans(c)) + ds = probs * (dp - d_i[:, None]) * scale + dq_lat += tl.dot(ds.to(c.dtype), c) + dq_rope += tl.dot(ds.to(kr.dtype), kr) + + tl.store( + dq_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + dq_lat.to(dq_lat_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_k[None, :], + ) + tl.store( + dq_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + dq_rope.to(dq_rope_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_dr[None, :], + ) + + @triton.jit + def _absorbed_mla_bwd_dkdv_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + dout_ptr, + lse_ptr, + d_ptr, + dc_part_ptr, + dkr_part_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_do_b, + stride_do_s, + stride_do_h, + stride_l_b, + stride_l_h, + stride_d_b, + stride_d_s, + stride_d_h, + stride_dc_g, + stride_dc_b, + stride_dc_t, + stride_dk_g, + stride_dk_b, + stride_dk_t, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + GROUP: tl.constexpr, + ): + pid_n = tl.program_id(0) + b = tl.program_id(1) + pid_g = tl.program_id(2) + head0 = pid_g * GROUP + + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_n = offs_n < T + mask_k = offs_k < K + mask_dr = offs_dr < DR + + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + dc = tl.zeros([BLOCK_N, BLOCK_K], tl.float32) + dkr = tl.zeros([BLOCK_N, BLOCK_DR], tl.float32) + + # Each program owns a disjoint head group, so the head reduction is + # written to a per-group workspace instead of racing through atomics. + for hi in range(0, GROUP): + h = head0 + hi + for start_m in range(0, S, BLOCK_M): + offs_m = start_m + tl.arange(0, BLOCK_M) + mask_m = offs_m < S + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + dout = tl.load( + dout_ptr + + b * stride_do_b + + offs_m[:, None] * stride_do_s + + h * stride_do_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + lse = tl.load( + lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, + mask=mask_m, + other=0.0, + ) + d_i = tl.load( + d_ptr + b * stride_d_b + offs_m * stride_d_s + h * stride_d_h, + mask=mask_m, + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + probs = tl.exp(scores - lse[:, None]) + probs = tl.where(mask_m[:, None] & mask_n[None, :], probs, 0.0) + + dp = tl.dot(dout, tl.trans(c)) + ds = probs * (dp - d_i[:, None]) * scale + dc += tl.dot(tl.trans(ds.to(c.dtype)), q_lat) + dc += tl.dot(tl.trans(probs.to(c.dtype)), dout) + dkr += tl.dot(tl.trans(ds.to(kr.dtype)), q_rope) + + base_dc = dc_part_ptr + pid_g * stride_dc_g + b * stride_dc_b + base_dk = dkr_part_ptr + pid_g * stride_dk_g + b * stride_dk_b + tl.store( + base_dc + offs_n[:, None] * stride_dc_t + offs_k[None, :], + dc, + mask=mask_n[:, None] & mask_k[None, :], + ) + tl.store( + base_dk + offs_n[:, None] * stride_dk_t + offs_dr[None, :], + dkr, + mask=mask_n[:, None] & mask_dr[None, :], + ) + + +def _require_fused_support(tensor: torch.Tensor) -> None: + if not _HAS_TRITON: + raise RuntimeError("fused absorbed MLA requires triton") + if not tensor.is_cuda: + raise RuntimeError("fused absorbed MLA requires CUDA tensors") + if tensor.dtype not in (torch.bfloat16, torch.float16): + raise RuntimeError( + "fused absorbed MLA supports bfloat16/float16, got " + f"{tensor.dtype}" + ) + + +class _AbsorbedMLAFusedCore(torch.autograd.Function): + @staticmethod + def forward(ctx, q_lat, q_rope, compressed_kv, k_rope, softmax_scale): + _require_fused_support(q_lat) + for name, tensor in ( + ("q_rope", q_rope), + ("compressed_kv", compressed_kv), + ("k_rope", k_rope), + ): + if tensor.dtype != q_lat.dtype: + raise ValueError(f"{name} dtype must match q_lat") + if q_lat.ndim != 4 or q_rope.ndim != 4 or compressed_kv.ndim != 3: + raise ValueError("fused absorbed MLA expects [B,S,H,K], [B,S,H,Dr], [B,T,K]") + if q_rope.shape[:3] != q_lat.shape[:3]: + raise ValueError("q_rope must match q_lat batch/query/head axes") + if k_rope.shape != (compressed_kv.size(0), compressed_kv.size(1), q_rope.size(-1)): + raise ValueError("k_rope must be [B,T,Dr]") + + q_lat = q_lat.contiguous() + q_rope = q_rope.contiguous() + compressed_kv = compressed_kv.contiguous() + k_rope = k_rope.contiguous() + batch, query_len, num_heads, key_dim = q_lat.shape + key_len = compressed_kv.size(1) + rope_dim = q_rope.size(-1) + + output = torch.empty_like(q_lat) + lse = torch.empty( + (batch, num_heads, query_len), + device=q_lat.device, + dtype=torch.float32, + ) + block_m, block_n, block_k, block_dr = _tile_sizes( + key_dim, rope_dim, _FWD_TILE_M, _FWD_TILE_N, query_len + ) + grid = (triton.cdiv(query_len, block_m), batch * num_heads) + _absorbed_mla_fwd_kernel[grid]( + q_lat, + q_rope, + compressed_kv, + k_rope, + output, + lse, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + softmax_scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + BLOCK_M=block_m, + BLOCK_N=block_n, + BLOCK_K=block_k, + BLOCK_DR=block_dr, + num_warps=_FWD_WARPS, + num_stages=_FWD_STAGES, + ) + ctx.save_for_backward(q_lat, q_rope, compressed_kv, k_rope, output, lse) + ctx.softmax_scale = float(softmax_scale) + return output + + @staticmethod + def backward(ctx, grad_output): + q_lat, q_rope, compressed_kv, k_rope, output, lse = ctx.saved_tensors + grad_output = grad_output.contiguous() + batch, query_len, num_heads, key_dim = q_lat.shape + key_len = compressed_kv.size(1) + rope_dim = q_rope.size(-1) + scale = ctx.softmax_scale + + d_i = (grad_output.float() * output.float()).sum(-1) + grad_q_lat = torch.empty_like(q_lat) + grad_q_rope = torch.empty_like(q_rope) + dq_m, dq_n, dq_k, dq_dr = _tile_sizes( + key_dim, rope_dim, _DQ_TILE_M, _DQ_TILE_N, query_len + ) + + grid_m = (triton.cdiv(query_len, dq_m), batch * num_heads) + _absorbed_mla_bwd_dq_kernel[grid_m]( + q_lat, + q_rope, + compressed_kv, + k_rope, + grad_output, + lse, + d_i, + grad_q_lat, + grad_q_rope, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + grad_output.stride(0), + grad_output.stride(1), + grad_output.stride(2), + lse.stride(0), + lse.stride(1), + d_i.stride(0), + d_i.stride(1), + d_i.stride(2), + BLOCK_M=dq_m, + BLOCK_N=dq_n, + BLOCK_K=dq_k, + BLOCK_DR=dq_dr, + num_warps=_DQ_WARPS, + num_stages=_DQ_STAGES, + ) + + dk_m, dk_n, dk_k, dk_dr = _tile_sizes( + key_dim, rope_dim, _DKDV_TILE_M, _DKDV_TILE_N, query_len + ) + group = _dkdv_group(num_heads) + num_groups = num_heads // group + dc_part = torch.zeros( + (num_groups, batch, key_len, key_dim), + device=q_lat.device, + dtype=torch.float32, + ) + dkr_part = torch.zeros( + (num_groups, batch, key_len, rope_dim), + device=q_lat.device, + dtype=torch.float32, + ) + grid_n = (triton.cdiv(key_len, dk_n), batch, num_groups) + _absorbed_mla_bwd_dkdv_kernel[grid_n]( + q_lat, + q_rope, + compressed_kv, + k_rope, + grad_output, + lse, + d_i, + dc_part, + dkr_part, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + grad_output.stride(0), + grad_output.stride(1), + grad_output.stride(2), + lse.stride(0), + lse.stride(1), + d_i.stride(0), + d_i.stride(1), + d_i.stride(2), + dc_part.stride(0), + dc_part.stride(1), + dc_part.stride(2), + dkr_part.stride(0), + dkr_part.stride(1), + dkr_part.stride(2), + BLOCK_M=dk_m, + BLOCK_N=dk_n, + BLOCK_K=dk_k, + BLOCK_DR=dk_dr, + GROUP=group, + num_warps=_DKDV_WARPS, + num_stages=_DKDV_STAGES, + ) + grad_c_kv = dc_part.sum(dim=0).to(compressed_kv.dtype) + grad_k_rope = dkr_part.sum(dim=0).to(k_rope.dtype) + return grad_q_lat, grad_q_rope, grad_c_kv, grad_k_rope, None + + +def fused_absorbed_mla_core( + q_lat: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + *, + softmax_scale: float, +) -> torch.Tensor: + """Fused absorbed attention core: ``[B,S,H,K] -> [B,S,H,K]``. + + ``softmax_scale`` is required because the core cannot recover + ``qk_nope_head_dim`` from the latent width alone. + """ + return _AbsorbedMLAFusedCore.apply( + q_lat, q_rope, compressed_kv, k_rope, float(softmax_scale) + ) + + +def fused_absorbed_mla_attention( + q_nope: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + kv_b_proj_weight: torch.Tensor, + *, + softmax_scale: float | None = None, + attn_mask: torch.Tensor | None = None, +) -> torch.Tensor: + """Absorbed MLA attention with a fused Triton score/softmax/latent core. + + ``softmax_scale`` defaults to ``1 / sqrt(Dn + Dr)`` where ``Dn`` is the + ``q_nope`` head dim and ``Dr`` the RoPE head dim, matching the materialized + MLA path. The fused core does not implement dense masks; a non-``None`` + ``attn_mask`` is rejected rather than silently ignored. + """ + if attn_mask is not None: + raise NotImplementedError( + "fused absorbed MLA does not support attn_mask; use " + "absorbed_mla_attention as the dense-mask reference" + ) + if q_nope.ndim != 4 or q_rope.ndim != 4: + raise ValueError("q_nope and q_rope must be [B, S, H, D]") + batch, query_len, num_heads, qk_nope_head_dim = q_nope.shape + qk_rope_head_dim = q_rope.size(-1) + if compressed_kv.ndim != 3 or compressed_kv.size(0) != batch: + raise ValueError("compressed_kv must be [B, T, K]") + if k_rope.shape != (batch, compressed_kv.size(1), qk_rope_head_dim): + raise ValueError("k_rope must be [B, T, qk_rope_head_dim]") + kv_lora_rank = compressed_kv.size(-1) + inferred_v_head_dim = kv_b_proj_weight.shape[0] // num_heads - qk_nope_head_dim + if inferred_v_head_dim <= 0 or kv_b_proj_weight.shape[0] % num_heads: + raise ValueError("kv_b_proj.weight rows do not describe per-head K/V") + w_uk, w_uv = split_kv_b_proj( + kv_b_proj_weight, + num_heads=num_heads, + qk_nope_head_dim=qk_nope_head_dim, + v_head_dim=inferred_v_head_dim, + kv_lora_rank=kv_lora_rank, + ) + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim) + + q_lat = torch.einsum("bshn,hnk->bshk", q_nope, w_uk) + out_latent = _AbsorbedMLAFusedCore.apply( + q_lat, q_rope, compressed_kv, k_rope, float(softmax_scale) + ) + output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) + return output.reshape(batch, query_len, num_heads * inferred_v_head_dim) diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 61321d2..cd27b4f 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -15,6 +15,7 @@ frameworks require separate native environments and their disclosed revisions. | `tools.performance.deepspec_offline_benchmark` | Native cache preparation and disk-fed training | | `tools.performance.bench_eagle3_train_core` | EAGLE3 train-core latency and memory | | `tools.performance.bench_thd_cp_attention` | Packed attention with context parallelism | +| `tools.performance.bench_absorbed_mla` | Absorbed versus materialized MLA attention, including the fused Triton core (one layer) | Run `python -m --help` for arguments. Templates are listed in the [configuration index](../../configs/README.md). Supply your own model, initial @@ -75,6 +76,8 @@ completion are included. - [DeepSpec](deepspec.md): resident and native cache-first workflows. - [EAGLE3](eagle3-h200.md): synthetic train-core measurement and numerical scope. - [Packed attention/CP](cudnn-thd-cp.md): attention component results. +- [Absorbed MLA](absorbed-mla.md): absorbed versus materialized attention identity and a negative training-shape result. +- [Absorbed MLA fused kernel](absorbed-mla-fused.md): fused Triton forward/backward for the absorbed identity, with memory and speed measurements. - [Training and acceptance](dspark-alignment.md): TorchSpec/ASpec BSHD/THD training alignment and verifier metrics. - [CP quality alignment](dspark-cp-alignment.md): CP2 versus same-layout CP1 diff --git a/docs/benchmarks/absorbed-mla-fused.md b/docs/benchmarks/absorbed-mla-fused.md new file mode 100644 index 0000000..4f1dfe8 --- /dev/null +++ b/docs/benchmarks/absorbed-mla-fused.md @@ -0,0 +1,95 @@ +# Absorbed MLA fused kernel (follow-up to #46) + +This note is the fused-kernel follow-up to +[absorbed-mla.md](absorbed-mla.md) and [ASpec #19](https://github.com/inclusionAI/ASpec/issues/19). +The dense reference in `aspec.models.ops.absorbed_mla` keeps `[B, H, S, T]` logits +materialized; the fused core keeps the same absorbed identity but never +materializes them: + +```text +q_lat = q_nope @ W_UK # [B, S, H, K] +logits = q_lat @ c_kv^T + q_rope @ k_rope^T +out = (softmax(logits) @ c_kv) @ W_UV^T +``` + +`aspec.models.ops.absorbed_mla_fused` adds a FlashAttention-style Triton +forward (online softmax accumulating the latent output) and a deterministic +backward. It is a prototype: dense masks, packed/THD layouts, Direct-CP and +`o_proj` fusion are not covered, and the naive reference remains the numerical +oracle in `tests/cpu/test_absorbed_mla.py`. The wrapper rejects a non-`None` +`attn_mask` with `NotImplementedError` instead of silently ignoring it. + +## Kernel design + +- Forward: one program per `[BLOCK_M, H]` query tile walks all keys, computes + `q_lat @ c_kv^T + q_rope @ k_rope^T`, and maintains `m`/`l`/`out_lat` in FP32. + Saved state is `out_lat` and `lse` only. +- Backward: two kernels, both recomputing logits from `lse`. + - query-side: `dq_lat`/`dq_rope` per query tile, no atomics; + - key/value-side: `dc_kv`/`dk_rope` per key tile and head group. `c_kv` is + shared across heads, so each program owns `H / GROUP` heads and writes a + per-group FP32 partial; a deterministic reduction sums the groups. This + avoids both the serial all-heads loop and atomics. +- The input projections (`q_nope @ W_UK`, `out_lat @ W_UV`) stay in PyTorch so + their gradients use normal autograd. +- BF16/FP16, contiguous BSHD/BTD inputs. The configured query tile is capped at + the draft block length to avoid wasted work on short blocks. + +Tiles were selected on the measurement host: forward `32x64`, query-side +`16x64`, key/value-side `64x32`, 4 warps; head group 8. + +## Measurement + +`tools.performance.bench_absorbed_mla` now includes `absorbed_fused` next to +`materialized_sdpa`, `materialized_explicit` and `absorbed`. It measures one +layer forward+backward at the attention boundary: BF16, `B=1`, `H=32`, +`Dn=128`, `Dr=64`, `Dv=128`, `K=512`, block 7, no mask, `o_proj`, Target +capture or optimizer step. Peak memory is `torch.cuda.max_memory_allocated()`. + +H100 PCIe 80GB, driver 580.126.09, PyTorch 2.13.0+cu130, Triton 3.7.1, warmup 2, +5 runs, median: + +| context | anchors | `S` | `T` | materialized SDPA | absorbed (PyTorch) | absorbed fused | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 4096 | 8 | 56 | 4152 | 3.37 ms / 0.245 GiB | 4.48 ms / 0.225 GiB | 3.86 ms / 0.137 GiB | +| 4096 | 128 | 896 | 4992 | 2.60 ms / 0.348 GiB | 10.30 ms / 2.799 GiB | 9.18 ms / 0.370 GiB | +| 4096 | 512 | 3584 | 7680 | 9.20 ms / 0.668 GiB | 63.64 ms / 16.688 GiB | 52.27 ms / 1.245 GiB | +| 16384 | 128 | 896 | 17280 | 6.39 ms / 0.830 GiB | 34.70 ms / 9.387 GiB | 28.59 ms / 0.449 GiB | +| 16384 | 256 | 1792 | 18176 | 11.04 ms / 0.937 GiB | 73.12 ms / 19.621 GiB | 59.30 ms / 0.675 GiB | + +## Result + +- The fused kernel is consistently faster than the naive PyTorch absorbed form + at every measured anchor budget (about 1.1–1.25x), and the gap grows with + `S` because the naive form materializes `[B, H, S, T]` logits. +- Peak memory collapses: 13.4x less than the naive absorbed form at the default + 512-anchor shape (1.245 vs 16.688 GiB) and 29x less at 16k context with 256 + anchors (0.675 vs 19.621 GiB). That is the main result: absorbed execution + trades extra FLOPs for activation memory, and a fused kernel makes the trade + usable. +- It is still slower than materialized SDPA (1.15–5.7x). The absorbed form does + about 3.4x more attention math at this geometry (`K+Dr` scores and `K` latent + output versus `192`/`128` materialized), and backward recomputes logits in two + passes. A kernel cannot remove that algorithmic gap on the default workload; + the memory reduction is the reason to consider it. + +Reproduce: + +```bash +env -u LD_LIBRARY_PATH CUDA_VISIBLE_DEVICES=1 \ + python -m tools.performance.bench_absorbed_mla \ + --context-length 4096 --num-anchors 512 --block-size 7 \ + --warmup 2 --runs 5 +``` + +Keep raw JSON outside the source tree. `LD_LIBRARY_PATH` was unset only to +select this PyTorch build's bundled cuDNN on the measurement host. + +## Remaining work + +The first gap versus the dense reference is boolean `attn_mask` support, +including fully masked rows; the fused wrapper currently fails closed on it. +Beyond that: FlexAttention/cuDNN masks, packed THD and Direct-CP layouts, +`o_proj` fusion, shape-based tile dispatch/autotuning, end-to-end memory/time +with Target capture, exact resume and export. None of these are established by +this note. diff --git a/docs/benchmarks/absorbed-mla.md b/docs/benchmarks/absorbed-mla.md new file mode 100644 index 0000000..f91547c --- /dev/null +++ b/docs/benchmarks/absorbed-mla.md @@ -0,0 +1,101 @@ +# Absorbed MLA feasibility (negative training result) + +This note is the feasibility milestone for [ASpec #19](https://github.com/inclusionAI/ASpec/issues/19). +It checks the absorbed-MLA identity on the DSpark draft geometry and measures whether +the naive absorbed form benefits draft training. It does **not** add a selectable +training path, a fused kernel or an export claim. + +## Derivation + +The retained materialized path computes, per draft layer, + +```text +k_nope = c_kv @ W_UK^T # [B, T, H, Dn] +v = c_kv @ W_UV^T # [B, T, H, Dv] +logits = q_nope @ k_nope^T + q_rope @ k_rope^T +out = softmax(logits) @ v +``` + +where `c_kv` is the post-norm latent KV (`kv_lora_rank`), `q_rope`/`k_rope` are the +RoPE side (`qk_rope_head_dim`) and `W_UK`/`W_UV` are the per-head blocks of +`kv_b_proj.weight`. Re-associating both products gives the absorbed form: + +```text +q_lat = q_nope @ W_UK # [B, S, H, K] +logits = q_lat @ c_kv^T + q_rope @ k_rope^T +out = (softmax(logits) @ c_kv) @ W_UV^T +``` + +RoPE still applies to the small side only, and both score terms are added before +the softmax. The identity is exact for the dual-source `cat(context, draft)` KV and +for `q_lora_rank` on/off: those projections happen before this boundary. + +## Scope of the prototype + +`aspec.models.ops.absorbed_mla.absorbed_mla_attention` implements the dense, +unmasked-or-boolean-mask case in PyTorch. Empty mask rows produce zero output and +zero gradient, matching fused SDPA. `tests/cpu/test_absorbed_mla.py` checks: + +- forward parity against an explicitly materialized reference; +- gradients of both query sides, `c_kv`, `k_rope` and `kv_b_proj.weight`; +- dense masks (including broadcast masks that hide keys and empty rows) and + short sequences; +- parity with the retained `K3DSparkMLAAttention` projections and softmax scale. + +FlexAttention/cuDNN masks, packed THD layouts, Direct-CP, `o_proj` fusion and +numerical policies for a fused kernel are not covered. + +## Measurement + +`tools/performance/bench_absorbed_mla.py` measures forward+backward at the attention +boundary for one layer: BF16, `B=1`, `H=32`, `Dn=128`, `Dr=64`, `Dv=128`, +`K=512`, context 4096, block 7. It compares the retained fused-SDPA path, an +explicit-softmax materialized path (same logits materialization as absorbed) and +the absorbed path. No mask, `o_proj`, Target capture or optimizer step is included. + +Measured on 8x H100 PCIe 80GB, driver 580.126.09, PyTorch 2.13.0+cu130, BF16, +warmup 1, 3 runs, median. Peak memory is `torch.cuda.max_memory_allocated()` for +forward+backward. Variants run in rotating order per round; the tool records each +round's order and the per-round timings and peaks in the JSON result. + +| anchors | `S` | `T` | materialized SDPA | materialized explicit | absorbed | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 7 | 4103 | 3.38 ms / 0.240 GiB | 4.02 ms / 0.184 GiB | 4.32 ms / 0.101 GiB | +| 8 | 56 | 4152 | 3.26 ms / 0.245 GiB | 3.60 ms / 0.258 GiB | 3.91 ms / 0.225 GiB | +| 128 | 896 | 4992 | 3.02 ms / 0.348 GiB | 9.04 ms / 2.315 GiB | 10.49 ms / 2.799 GiB | +| 512 | 3584 | 7680 | 9.14 ms / 0.668 GiB | 51.50 ms / 13.411 GiB | 63.11 ms / 16.688 GiB | + +`S` is `num_anchors * block_size`; `T = context_length + S`. + +Reproduce with: + +```bash +env -u LD_LIBRARY_PATH CUDA_VISIBLE_DEVICES=1 \ + python -m tools.performance.bench_absorbed_mla \ + --context-length 4096 --num-anchors 512 --block-size 7 \ + --warmup 1 --runs 3 --result /tmp/absorbed-mla.json +``` + +Keep the raw JSON outside the source tree. `LD_LIBRARY_PATH` was unset only to +select this PyTorch build's bundled cuDNN on the measurement host. + +## Result and recommendation + +For the default draft anchor budget (512 anchors, 3584 queries), the naive absorbed +path is both slower and larger than the retained fused-SDPA path: explicit logits +dominate memory, and `q_lat` has `kv_lora_rank=512` versus the materialized +`qk_nope_head_dim=128`, so the score matmul does more work. The tiny-query +regimes (1 or 8 anchors) do reduce peak activation memory (0.101 vs 0.240 GiB at +one anchor, 0.225 vs 0.245 GiB at eight), but both are slower. + +**Recommendation:** do not integrate the naive PyTorch absorbed form into maintained +draft training, and do not claim a training speedup. A useful follow-up would need +a fused absorbed kernel (score + softmax + latent output) or a workload where the +query/key ratio is genuinely tiny; both are separate from this prototype. The +identity and gradient checks remain useful for validating such a kernel. + +## Remaining work + +Fused forward/backward kernels, FlexAttention/cuDNN mask support, packing and +Direct-CP layouts, `o_proj` fusion, end-to-end memory/time including target feature +capture, exact resume and export. None of these are established by this note. diff --git a/tests/cpu/test_absorbed_mla.py b/tests/cpu/test_absorbed_mla.py new file mode 100644 index 0000000..1e3dd2f --- /dev/null +++ b/tests/cpu/test_absorbed_mla.py @@ -0,0 +1,269 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Absorbed-MLA feasibility checks. + +The reference expands ``kv_b_proj`` into per-head K/V and runs an explicit +materialized attention. These checks cover the forward identity, affected +gradients, dense masks and the DSpark MLA geometry; they do not establish a +production speedup. +""" + +import math + +import pytest +import torch + +from aspec.models.draft.dspark import ( + K3DSparkConfig, + K3DSparkMLAAttention, + _apply_rope_interleaved, +) +from aspec.models.ops.absorbed_mla import absorbed_mla_attention, split_kv_b_proj + + +def make_tensors(batch=1, query=4, key=7, heads=2, nope=4, rope=2, value=3, latent=5): + generator = torch.Generator().manual_seed(23) + return dict( + q_nope=torch.randn(batch, query, heads, nope, generator=generator), + q_rope=torch.randn(batch, query, heads, rope, generator=generator), + compressed_kv=torch.randn(batch, key, latent, generator=generator), + k_rope=torch.randn(batch, key, rope, generator=generator), + kv_b_proj_weight=torch.randn( + heads * (nope + value), latent, generator=generator + ), + ) + + +def reference_materialized( + q_nope, q_rope, compressed_kv, k_rope, kv_b_proj_weight, + *, softmax_scale=None, attn_mask=None, +): + """Explicitly materialize per-head K/V, then run dense attention.""" + batch, query, heads, nope = q_nope.shape + rope = q_rope.size(-1) + value = kv_b_proj_weight.shape[0] // heads - nope + latent = compressed_kv.size(-1) + view = kv_b_proj_weight.view(heads, nope + value, latent) + w_uk, w_uv = view[:, :nope, :], view[:, nope:, :] + k_nope = torch.einsum("btk,hnk->bthn", compressed_kv, w_uk) + values = torch.einsum("btk,hvk->bthv", compressed_kv, w_uv) + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(nope + rope) + scores = torch.einsum("bshn,bthn->bhst", q_nope, k_nope) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope, k_rope) + scores = scores * softmax_scale + if attn_mask is not None: + mask = attn_mask.expand(scores.shape) + scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores.float(), dim=-1) + if attn_mask is not None: + probs = probs * mask.any(dim=-1, keepdim=True) + probs = probs.to(q_nope.dtype) + output = torch.einsum("bhst,bthv->bshv", probs, values) + return output.reshape(batch, query, heads * value) + + +def make_attention_config(**overrides): + values = dict( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=2, vocab_size=32, + q_lora_rank=8, kv_lora_rank=5, qk_nope_head_dim=4, + qk_rope_head_dim=2, v_head_dim=3, num_target_layers=1, + target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=64, + rms_norm_eps=1e-6, rope_theta=10000.0, + ) + values.update(overrides) + return K3DSparkConfig(**values) + + +def project_q_and_latent(attention, draft_hidden, context_hidden, + draft_position_ids, context_position_ids): + """Reproduce the materialized MLA boundary before attention.""" + batch, draft_len, _ = draft_hidden.shape + if attention.q_lora_rank is not None: + query = attention.q_b_proj( + attention.q_a_layernorm(attention.q_a_proj(draft_hidden)) + ) + else: + query = attention.q_proj(draft_hidden) + query = query.view(batch, draft_len, attention.num_heads, attention.qk_head_dim) + # The retained path transposes to [B, H, S, D] before RoPE. + query = query.transpose(1, 2) + q_nope, q_rope = torch.split( + query, [attention.qk_nope_head_dim, attention.qk_rope_head_dim], dim=-1 + ) + kv_input = torch.cat([context_hidden, draft_hidden], dim=1) + kv_combined = attention.kv_a_proj_with_mqa(kv_input) + compressed, k_rope = torch.split( + kv_combined, [attention.kv_lora_rank, attention.qk_rope_head_dim], dim=-1 + ) + compressed = attention.kv_a_layernorm(compressed) + k_rope = k_rope.unsqueeze(1) + positions = torch.cat([context_position_ids, draft_position_ids], dim=1) + cos, sin = attention.rotary_emb(q_rope, seq_len=positions.size(1)) + q_rope = _apply_rope_interleaved( + q_rope, cos, sin, draft_position_ids, unsqueeze_dim=1 + ) + k_rope = _apply_rope_interleaved( + k_rope, cos, sin, positions, unsqueeze_dim=1 + ) + # The materialized path keeps [B, H, T, D]; the absorbed reference takes + # the BSHD-latent boundary, so K rope is the shared [B, T, D] tensor. + return ( + q_nope.transpose(1, 2), + q_rope.transpose(1, 2), + compressed, + k_rope.squeeze(1), + ) + + +def test_absorbed_forward_matches_materialized_equations(): + tensors = make_tensors() + absorbed = absorbed_mla_attention(**tensors) + expected = reference_materialized(**tensors) + assert absorbed.shape == (1, 4, 6) + torch.testing.assert_close(absorbed, expected, rtol=2e-6, atol=2e-6) + + +@pytest.mark.parametrize("materialized_first", [True, False]) +def test_absorbed_gradients_match_materialized(materialized_first): + tensors = make_tensors(batch=2) + reference_tensors = {name: value.clone() for name, value in tensors.items()} + for value in tensors.values(): + value.requires_grad_(True) + for value in reference_tensors.values(): + value.requires_grad_(True) + + def run(fn, values): + output = fn(**values) + loss = output.float().pow(2).sum() + return torch.autograd.grad( + loss, [*values.values()], allow_unused=False + ) + + if materialized_first: + expected = run(reference_materialized, reference_tensors) + actual = run(absorbed_mla_attention, tensors) + else: + actual = run(absorbed_mla_attention, tensors) + expected = run(reference_materialized, reference_tensors) + for name, actual_grad, expected_grad in zip( + tensors, actual, expected, strict=True + ): + torch.testing.assert_close( + actual_grad, expected_grad, rtol=1e-5, atol=2e-5, msg=name + ) + + +def test_dense_mask_excludes_hidden_keys(): + tensors = make_tensors() + mask = torch.ones(1, 2, 4, 7, dtype=torch.bool) + mask[:, :, :, 5:] = False + expected = reference_materialized(**tensors, attn_mask=mask) + actual = absorbed_mla_attention(**tensors, attn_mask=mask) + torch.testing.assert_close(actual, expected, rtol=2e-6, atol=2e-6) + + changed = {name: value.clone() for name, value in tensors.items()} + changed["compressed_kv"][:, 5:] += 100.0 + changed["k_rope"][:, 5:] -= 100.0 + moved = absorbed_mla_attention(**changed, attn_mask=mask) + torch.testing.assert_close(moved, actual, rtol=0, atol=0) + + +def test_fully_masked_rows_have_zero_output_and_gradient(): + tensors = make_tensors() + for value in tensors.values(): + value.requires_grad_(True) + mask = torch.ones(1, 2, 4, 7, dtype=torch.bool) + mask[:, :, :2] = False + output = absorbed_mla_attention(**tensors, attn_mask=mask) + assert torch.count_nonzero(output[:, :2]) == 0 + grads = torch.autograd.grad(output.float().pow(2).sum(), list(tensors.values())) + q_nope_grad = grads[0] + assert torch.count_nonzero(q_nope_grad[:, :2]) == 0 + expected = reference_materialized(**tensors, attn_mask=mask) + torch.testing.assert_close(output, expected, rtol=2e-6, atol=2e-6) + + +def test_mask_broadcasts_over_batch_and_heads(): + tensors = make_tensors(batch=2) + mask = torch.tril(torch.ones(4, 7, dtype=torch.bool)) + actual = absorbed_mla_attention(**tensors, attn_mask=mask) + expected = reference_materialized(**tensors, attn_mask=mask) + torch.testing.assert_close(actual, expected, rtol=2e-6, atol=2e-6) + + +def test_short_and_single_token_sequences(): + tensors = make_tensors(query=1, key=1) + torch.testing.assert_close( + absorbed_mla_attention(**tensors), reference_materialized(**tensors), + rtol=2e-6, atol=2e-6, + ) + tensors = make_tensors(query=1, key=3) + torch.testing.assert_close( + absorbed_mla_attention(**tensors), reference_materialized(**tensors), + rtol=2e-6, atol=2e-6, + ) + + +def test_split_rejects_wrong_weight_shape(): + with pytest.raises(ValueError, match="kv_b_proj.weight"): + split_kv_b_proj( + torch.zeros(13, 5), num_heads=2, qk_nope_head_dim=4, + v_head_dim=3, kv_lora_rank=5, + ) + + +@pytest.mark.parametrize("mutation,match", [ + (lambda t: t.update(k_rope=t["k_rope"][:, :2]), "k_rope"), + (lambda t: t.update(k_rope=t["k_rope"].double()), "dtype"), + (lambda t: t.update(attn_mask=torch.ones(4, 7)), "boolean"), + (lambda t: t.update(attn_mask=torch.ones(1, 3, 4, 7, dtype=torch.bool)), "broadcast"), + (lambda t: t.update(softmax_scale=0.0), "softmax_scale"), + (lambda t: t.update(softmax_scale=float("nan")), "softmax_scale"), + (lambda t: t.update(softmax_scale=True), "softmax_scale"), +]) +def test_invalid_inputs_fail_closed(mutation, match): + tensors = dict(make_tensors(), softmax_scale=None, attn_mask=None) + tensors.pop("softmax_scale") + tensors.pop("attn_mask") + mutation(tensors) + with pytest.raises(ValueError, match=match): + absorbed_mla_attention(**tensors) + + +@pytest.mark.parametrize("q_lora_rank", [8, None]) +def test_matches_the_retained_materialized_dspark_attention(q_lora_rank): + torch.manual_seed(11) + attention = K3DSparkMLAAttention( + make_attention_config(q_lora_rank=q_lora_rank) + ) + batch, query, context = 1, 3, 6 + draft_hidden = torch.randn(batch, query, 16) + context_hidden = torch.randn(batch, context, 16) + draft_positions = torch.arange(context, context + query).unsqueeze(0) + context_positions = torch.arange(context).unsqueeze(0) + + retained = attention( + draft_hidden, context_hidden, draft_positions, context_positions, + block_mask=None, + ) + q_nope, q_rope, compressed, k_rope = project_q_and_latent( + attention, draft_hidden, context_hidden, draft_positions, context_positions + ) + reference = reference_materialized( + q_nope, q_rope, compressed, k_rope, attention.kv_b_proj.weight, + softmax_scale=attention.softmax_scale, + ) + absorbed = absorbed_mla_attention( + q_nope, q_rope, compressed, k_rope, attention.kv_b_proj.weight, + softmax_scale=attention.softmax_scale, + ) + torch.testing.assert_close( + attention.o_proj(reference), retained, rtol=2e-6, atol=2e-6 + ) + torch.testing.assert_close( + attention.o_proj(absorbed), retained, rtol=2e-6, atol=2e-6 + ) diff --git a/tests/test_absorbed_mla_fused_gpu.py b/tests/test_absorbed_mla_fused_gpu.py new file mode 100644 index 0000000..9061ef4 --- /dev/null +++ b/tests/test_absorbed_mla_fused_gpu.py @@ -0,0 +1,148 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU checks for the fused Triton absorbed-MLA attention core. + +These tests compare the fused kernel against the dense PyTorch reference in +``aspec.models.ops.absorbed_mla``. They require a CUDA device and Triton; the +CPU suite skips them. +""" + +import math + +import pytest +import torch + +from aspec.models.ops import absorbed_mla_fused as fused_ops +from aspec.models.ops.absorbed_mla import absorbed_mla_attention + +pytestmark = pytest.mark.skipif( + not fused_ops._HAS_TRITON or not torch.cuda.is_available(), + reason="fused absorbed MLA requires Triton and CUDA", +) + + +def _make_tensors(*, batch=1, query=8, key=41, heads=4, nope=16, rope=16, value=16, latent=32): + generator = torch.Generator().manual_seed(41) + return { + "q_nope": torch.randn( + batch, query, heads, nope, generator=generator + ).cuda().half(), + "q_rope": torch.randn( + batch, query, heads, rope, generator=generator + ).cuda().half(), + "compressed_kv": torch.randn( + batch, key, latent, generator=generator + ).cuda().half(), + "k_rope": torch.randn(batch, key, rope, generator=generator).cuda().half(), + "kv_b_proj_weight": torch.randn( + heads * (nope + value), latent, generator=generator + ).cuda().half(), + } + + +def _relative_error(actual, expected): + expected = expected.float() + return ( + (actual.float() - expected).norm() / expected.norm().clamp_min(1e-6) + ).item() + + +def _run_grads(fn, tensors): + values = {name: value.detach().clone().requires_grad_(True) for name, value in tensors.items()} + output = fn(values) + grads = torch.autograd.grad(output.float().pow(2).sum(), list(values.values())) + return output, grads + + +@pytest.mark.parametrize( + "shape", + [ + dict(batch=1, query=8, key=41, heads=2), + dict(batch=1, query=7, key=41, heads=3), + dict(batch=2, query=33, key=100, heads=8, latent=64), + dict(batch=1, query=70, key=200, heads=4, latent=128), + ], +) +def test_fused_forward_and_gradients_match_reference(shape): + tensors = _make_tensors(**shape) + reference = absorbed_mla_attention(**tensors) + fused = fused_ops.fused_absorbed_mla_attention(**tensors) + assert torch.isfinite(fused).all() + assert _relative_error(fused, reference) < 2e-2 + + _, reference_grads = _run_grads( + lambda values: absorbed_mla_attention(**values), tensors + ) + _, fused_grads = _run_grads( + lambda values: fused_ops.fused_absorbed_mla_attention(**values), tensors + ) + for name, expected, actual in zip( + tensors, reference_grads, fused_grads, strict=True + ): + assert torch.isfinite(actual).all(), name + assert _relative_error(actual, expected) < 3e-2, name + + +def test_fused_default_scale_matches_materialized(): + tensors = _make_tensors() + fused = fused_ops.fused_absorbed_mla_attention(**tensors) + explicit = fused_ops.fused_absorbed_mla_attention( + **tensors, + softmax_scale=1.0 / math.sqrt(16 + 16), + ) + torch.testing.assert_close(fused, explicit, rtol=0, atol=0) + + +def test_fused_core_matches_float32_reference_at_latent_512(): + torch.manual_seed(7) + query, key, heads, latent, rope = 8, 64, 2, 512, 64 + q_lat = torch.randn(1, query, heads, latent, device="cuda", dtype=torch.bfloat16) + q_rope = torch.randn(1, query, heads, rope, device="cuda", dtype=torch.bfloat16) + c_kv = torch.randn(1, key, latent, device="cuda", dtype=torch.bfloat16) + k_rope = torch.randn(1, key, rope, device="cuda", dtype=torch.bfloat16) + scale = 1.0 / math.sqrt(128 + 64) + + def reference(q_lat, q_rope, c_kv, k_rope): + scores = torch.einsum("bshk,btk->bhst", q_lat.float(), c_kv.float()) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope.float(), k_rope.float()) + probs = torch.softmax(scores * scale, dim=-1) + return torch.einsum("bhst,btk->bshk", probs, c_kv.float()) + + values = [q_lat, q_rope, c_kv, k_rope] + expected = reference(*values) + actual = fused_ops.fused_absorbed_mla_core( + q_lat, q_rope, c_kv, k_rope, softmax_scale=scale + ) + assert torch.isfinite(actual).all() + assert _relative_error(actual, expected) < 1e-2 + + fused_values = [value.detach().clone().requires_grad_(True) for value in values] + ref_values = [value.detach().clone().requires_grad_(True) for value in values] + ref_out = reference(*ref_values) + fused_out = fused_ops.fused_absorbed_mla_core( + *fused_values, softmax_scale=scale + ) + ref_grads = torch.autograd.grad(ref_out.pow(2).sum(), ref_values) + fused_grads = torch.autograd.grad(fused_out.float().pow(2).sum(), fused_values) + for expected_grad, actual_grad in zip(ref_grads, fused_grads, strict=True): + assert torch.isfinite(actual_grad).all() + assert _relative_error(actual_grad, expected_grad) < 3e-2 + + +def test_fused_core_rejects_cpu_tensors(): + if not fused_ops._HAS_TRITON: + pytest.skip("triton is not installed") + tensor = torch.zeros(1, 2, 1, 16) + with pytest.raises(RuntimeError, match="CUDA"): + fused_ops.fused_absorbed_mla_core( + tensor, tensor, torch.zeros(1, 2, 16), torch.zeros(1, 2, 16), + softmax_scale=0.1, + ) + + +def test_fused_rejects_dense_mask(): + tensors = _make_tensors() + mask = torch.ones(1, 8, 4, 41, dtype=torch.bool, device="cuda") + with pytest.raises(NotImplementedError, match="attn_mask"): + fused_ops.fused_absorbed_mla_attention(**tensors, attn_mask=mask) diff --git a/tools/performance/bench_absorbed_mla.py b/tools/performance/bench_absorbed_mla.py new file mode 100644 index 0000000..b3eb101 --- /dev/null +++ b/tools/performance/bench_absorbed_mla.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Feasibility benchmark for absorbed versus materialized MLA attention. + +The workload starts at the DSpark MLA attention boundary (BSHD query/latent +tensors, BF16, no dense mask). It compares: + +- ``materialized_sdpa``: expand ``kv_b_proj`` and call fused SDPA; +- ``materialized_explicit``: expand ``kv_b_proj`` and materialize logits; +- ``absorbed``: keep the normalized latent and re-associate both projections; +- ``absorbed_fused``: the same absorbed identity with the fused Triton + score/softmax/latent core (when Triton and CUDA are available). + +FlexAttention/cuDNN masks, packed layouts and Direct-CP are outside this +benchmark. Report the boundary together with the result; this does not establish +end-to-end training throughput. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +import torch +import torch.nn.functional as F + +from aspec.models.ops.absorbed_mla import absorbed_mla_attention +from aspec.models.ops.absorbed_mla_fused import ( + _HAS_TRITON, + fused_absorbed_mla_attention, +) + + +@dataclass(frozen=True) +class Workload: + context_length: int = 4096 + num_anchors: int = 512 + block_size: int = 7 + num_heads: int = 32 + qk_nope_head_dim: int = 128 + qk_rope_head_dim: int = 64 + v_head_dim: int = 128 + kv_lora_rank: int = 512 + batch_size: int = 1 + + @property + def query_length(self) -> int: + return self.num_anchors * self.block_size + + @property + def key_length(self) -> int: + return self.context_length + self.query_length + + @property + def softmax_scale(self) -> float: + return 1.0 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim) + + +def build_inputs(workload: Workload, device: torch.device, dtype: torch.dtype): + generator = torch.Generator(device="cpu").manual_seed(7) + shape = (workload.batch_size, workload.query_length, workload.num_heads) + tensors = { + "q_nope": torch.randn( + *shape, workload.qk_nope_head_dim, generator=generator + ).to(device=device, dtype=dtype), + "q_rope": torch.randn( + *shape, workload.qk_rope_head_dim, generator=generator + ).to(device=device, dtype=dtype), + "compressed_kv": torch.randn( + workload.batch_size, workload.key_length, workload.kv_lora_rank, + generator=generator, + ).to(device=device, dtype=dtype), + "k_rope": torch.randn( + workload.batch_size, workload.key_length, workload.qk_rope_head_dim, + generator=generator, + ).to(device=device, dtype=dtype), + "kv_b_proj_weight": torch.randn( + workload.num_heads + * (workload.qk_nope_head_dim + workload.v_head_dim), + workload.kv_lora_rank, + generator=generator, + ).to(device=device, dtype=dtype), + } + for tensor in tensors.values(): + tensor.requires_grad_(True) + return tensors + + +def split_projection(weight, workload: Workload): + view = weight.view( + workload.num_heads, + workload.qk_nope_head_dim + workload.v_head_dim, + workload.kv_lora_rank, + ) + return view[:, : workload.qk_nope_head_dim], view[:, workload.qk_nope_head_dim :] + + +def materialized_kv(compressed_kv, weight, workload: Workload): + w_uk, w_uv = split_projection(weight, workload) + k_nope = torch.einsum("btk,hnk->bthn", compressed_kv, w_uk) + values = torch.einsum("btk,hvk->bthv", compressed_kv, w_uv) + return k_nope, values + + +def materialized_sdpa(tensors, workload: Workload): + q_nope = tensors["q_nope"].transpose(1, 2) + q_rope = tensors["q_rope"].transpose(1, 2) + k_nope, values = materialized_kv( + tensors["compressed_kv"], tensors["kv_b_proj_weight"], workload + ) + key = torch.cat( + [ + k_nope.transpose(1, 2), + tensors["k_rope"].unsqueeze(1).expand(-1, workload.num_heads, -1, -1), + ], + dim=-1, + ) + query = torch.cat([q_nope, q_rope], dim=-1) + output = F.scaled_dot_product_attention( + query, key, values.transpose(1, 2), is_causal=False, dropout_p=0.0, + scale=workload.softmax_scale, + ) + return output.transpose(1, 2).reshape( + workload.batch_size, workload.query_length, + workload.num_heads * workload.v_head_dim, + ) + + +def materialized_explicit(tensors, workload: Workload): + q_nope = tensors["q_nope"] + k_nope, values = materialized_kv( + tensors["compressed_kv"], tensors["kv_b_proj_weight"], workload + ) + scores = torch.einsum("bshn,bthn->bhst", q_nope, k_nope) + scores = scores + torch.einsum( + "bshr,btr->bhst", tensors["q_rope"], tensors["k_rope"] + ) + scores = scores * workload.softmax_scale + probs = torch.softmax(scores.float(), dim=-1).to(q_nope.dtype) + output = torch.einsum("bhst,bthv->bshv", probs, values) + return output.reshape( + workload.batch_size, workload.query_length, + workload.num_heads * workload.v_head_dim, + ) + + +def absorbed(tensors, workload: Workload): + return absorbed_mla_attention( + tensors["q_nope"], tensors["q_rope"], tensors["compressed_kv"], + tensors["k_rope"], tensors["kv_b_proj_weight"], + softmax_scale=workload.softmax_scale, + ) + + +def absorbed_fused(tensors, workload: Workload): + return fused_absorbed_mla_attention( + tensors["q_nope"], tensors["q_rope"], tensors["compressed_kv"], + tensors["k_rope"], tensors["kv_b_proj_weight"], + softmax_scale=workload.softmax_scale, + ) + + +def run_step(fn, tensors, workload): + output = fn(tensors, workload) + loss = output.float().pow(2).sum() + return torch.autograd.grad(loss, list(tensors.values())) + + +def measure_variants(variants, tensors, workload, *, warmup: int, runs: int): + """Time each variant in rotating order and record per-round raw samples.""" + names = list(variants) + for round_index in range(warmup): + order = names[round_index % len(names):] + names[: round_index % len(names)] + for name in order: + run_step(variants[name], tensors, workload) + torch.cuda.synchronize() + samples = {name: [] for name in names} + peaks = {name: [] for name in names} + orders = [] + for round_index in range(runs): + offset = round_index % len(names) + order = names[offset:] + names[:offset] + orders.append(order) + for name in order: + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + run_step(variants[name], tensors, workload) + end.record() + end.synchronize() + samples[name].append(start.elapsed_time(end)) + peaks[name].append(torch.cuda.max_memory_allocated()) + results = {} + for name in names: + results[name] = { + "median_ms": statistics.median(samples[name]), + "min_ms": min(samples[name]), + "max_ms": max(samples[name]), + "samples_ms": samples[name], + "peak_memory_samples_bytes": peaks[name], + "peak_memory_bytes": max(peaks[name]), + } + return results, orders + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--context-length", type=int, default=4096) + parser.add_argument("--num-anchors", type=int, default=512) + parser.add_argument("--block-size", type=int, default=7) + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16") + parser.add_argument("--result", type=Path) + args = parser.parse_args(argv) + if not torch.cuda.is_available(): + raise SystemExit("absorbed MLA benchmark requires CUDA") + if args.warmup < 1 or args.runs < 1: + raise SystemExit("warmup and runs must be positive") + workload = Workload( + context_length=args.context_length, + num_anchors=args.num_anchors, + block_size=args.block_size, + batch_size=args.batch_size, + ) + dtype = getattr(torch, args.dtype) + device = torch.device("cuda") + tensors = build_inputs(workload, device, dtype) + variants = { + "materialized_sdpa": materialized_sdpa, + "materialized_explicit": materialized_explicit, + "absorbed": absorbed, + } + if _HAS_TRITON: + variants["absorbed_fused"] = absorbed_fused + results = { + "workload": asdict(workload) | { + "query_length": workload.query_length, + "key_length": workload.key_length, + "dtype": args.dtype, + }, + "device": torch.cuda.get_device_name(), + "torch": torch.__version__, + "boundary": "attention core, dense unmasked, forward+backward, no o_proj", + "variants": {}, + } + measured, orders = measure_variants( + variants, tensors, workload, warmup=args.warmup, runs=args.runs + ) + results["variant_order"] = orders + results["variants"] = measured + for name, value in measured.items(): + print(name, json.dumps(value)) + if args.result: + args.result.write_text(json.dumps(results, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main())