From 2f3e410f15aef98c61b70150976be78cfafb8d28 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:14:19 +0800 Subject: [PATCH 1/4] feat(mla): add absorbed attention feasibility prototype Add a dense absorbed-MLA reference for the DSpark dual-source geometry, numerical/gradient checks against the materialized path, and a one-layer benchmark tool. The measured default-anchor training shape shows no speed or memory benefit from the naive absorbed form, so it stays out of the training path; the checks are reusable for a future fused kernel. --- aspec/models/ops/absorbed_mla.py | 147 ++++++++++++++ docs/benchmarks/README.md | 2 + docs/benchmarks/absorbed-mla.md | 98 +++++++++ tests/cpu/test_absorbed_mla.py | 243 +++++++++++++++++++++++ tools/performance/bench_absorbed_mla.py | 251 ++++++++++++++++++++++++ 5 files changed, 741 insertions(+) create mode 100644 aspec/models/ops/absorbed_mla.py create mode 100644 docs/benchmarks/absorbed-mla.md create mode 100644 tests/cpu/test_absorbed_mla.py create mode 100644 tools/performance/bench_absorbed_mla.py diff --git a/aspec/models/ops/absorbed_mla.py b/aspec/models/ops/absorbed_mla.py new file mode 100644 index 0000000..c1e5686 --- /dev/null +++ b/aspec/models/ops/absorbed_mla.py @@ -0,0 +1,147 @@ +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reference absorbed-MLA attention for feasibility checks. + +The standard DeepSeek-V2/V3 identity re-associates the materialized MLA +projections: + + q_nope @ (c_kv @ W_UK^T)^T == (q_nope @ W_UK) @ c_kv^T + sum_t p_t (c_kv_t @ W_UV^T) == (sum_t p_t c_kv_t) @ W_UV^T + +so the attention boundary can consume the normalized latent ``c_kv`` directly +instead of materializing per-head ``k_nope`` and ``v``. RoPE is still applied to +the small ``qk_rope_head_dim`` side, and the two score terms are added before +the softmax. + +This module is a dense-mask reference implementation used by numerical and +feasibility checks. It is not wired into training, and it does not implement the +FlexAttention, cuDNN THD, Direct-CP or fused-kernel paths needed for a +production claim. Dense boolean masks are broadcast against ``[B, H, S, T]``. +""" + +from __future__ import annotations + +import math + +import torch + + +def split_kv_b_proj( + weight: torch.Tensor, + *, + num_heads: int, + qk_nope_head_dim: int, + v_head_dim: int, + kv_lora_rank: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Split ``kv_b_proj.weight`` into per-head ``W_UK`` and ``W_UV`` views.""" + expected = (num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank) + if tuple(weight.shape) != expected: + raise ValueError( + "kv_b_proj.weight must have shape " + f"{expected}, got {tuple(weight.shape)}" + ) + view = weight.view(num_heads, qk_nope_head_dim + v_head_dim, kv_lora_rank) + return view[:, :qk_nope_head_dim, :], view[:, qk_nope_head_dim:, :] + + +def absorbed_mla_attention( + q_nope: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + kv_b_proj_weight: torch.Tensor, + *, + softmax_scale: float | None = None, + attn_mask: torch.Tensor | None = None, +) -> torch.Tensor: + """Absorbed MLA attention with a dense mask. + + Args: + q_nope: ``[B, S, H, Dn]`` query without RoPE. + q_rope: ``[B, S, H, Dr]`` query after RoPE. + compressed_kv: ``[B, T, K]`` normalized latent KV before ``kv_b_proj``. + k_rope: ``[B, T, Dr]`` shared MQA key after RoPE. + kv_b_proj_weight: ``[H * (Dn + Dv), K]`` materialized projection weight. + softmax_scale: score scale; defaults to ``1 / sqrt(Dn + Dr)``. + attn_mask: optional boolean mask broadcastable to ``[B, H, S, T]``. + + Returns: + ``[B, S, H * Dv]`` attention output before ``o_proj``. + """ + if q_nope.ndim != 4 or q_rope.ndim != 4: + raise ValueError("q_nope and q_rope must be [B, S, H, D]") + batch, query_len, num_heads, qk_nope_head_dim = q_nope.shape + if q_rope.shape[:3] != q_nope.shape[:3]: + raise ValueError("q_rope must match q_nope's batch, query and head axes") + qk_rope_head_dim = q_rope.size(-1) + if compressed_kv.ndim != 3: + raise ValueError("compressed_kv must be [B, T, K]") + if compressed_kv.shape[0] != batch: + raise ValueError("compressed_kv batch size must match the queries") + key_len, kv_lora_rank = compressed_kv.shape[1], compressed_kv.shape[2] + if k_rope.shape != (batch, key_len, qk_rope_head_dim): + raise ValueError( + "k_rope must be [B, T, qk_rope_head_dim] with the compressed_kv " + "batch and key length" + ) + if kv_b_proj_weight.ndim != 2: + raise ValueError("kv_b_proj.weight must be a matrix") + inferred_v_head_dim = ( + kv_b_proj_weight.shape[0] // num_heads - qk_nope_head_dim + ) + if inferred_v_head_dim <= 0 or kv_b_proj_weight.shape[0] % num_heads: + raise ValueError("kv_b_proj.weight rows do not describe per-head K/V") + w_uk, w_uv = split_kv_b_proj( + kv_b_proj_weight, + num_heads=num_heads, + qk_nope_head_dim=qk_nope_head_dim, + v_head_dim=inferred_v_head_dim, + kv_lora_rank=kv_lora_rank, + ) + + dtype = q_nope.dtype + for name, tensor in ( + ("q_rope", q_rope), + ("compressed_kv", compressed_kv), + ("k_rope", k_rope), + ("kv_b_proj.weight", kv_b_proj_weight), + ): + if tensor.dtype != dtype: + raise ValueError(f"{name} dtype must match q_nope ({dtype})") + if not dtype.is_floating_point: + raise ValueError("absorbed MLA requires floating-point tensors") + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim) + if not math.isfinite(softmax_scale) or softmax_scale <= 0: + raise ValueError("softmax_scale must be positive and finite") + + q_latent = torch.einsum("bshn,hnk->bshk", q_nope, w_uk) + scores = torch.einsum("bshk,btk->bhst", q_latent, compressed_kv) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope, k_rope) + scores = scores * softmax_scale + if attn_mask is not None: + if attn_mask.dtype != torch.bool: + raise ValueError("attn_mask must be boolean") + try: + scores = scores.masked_fill(~attn_mask, torch.finfo(scores.dtype).min) + except RuntimeError as exc: + raise ValueError( + "attn_mask must broadcast against [B, H, S, T]" + ) from exc + probs = torch.softmax(scores.float(), dim=-1).to(dtype) + out_latent = torch.einsum("bhst,btk->bshk", probs, compressed_kv) + output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) + return output.reshape(batch, query_len, num_heads * inferred_v_head_dim) diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 61321d2..ca61cad 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -15,6 +15,7 @@ frameworks require separate native environments and their disclosed revisions. | `tools.performance.deepspec_offline_benchmark` | Native cache preparation and disk-fed training | | `tools.performance.bench_eagle3_train_core` | EAGLE3 train-core latency and memory | | `tools.performance.bench_thd_cp_attention` | Packed attention with context parallelism | +| `tools.performance.bench_absorbed_mla` | Absorbed versus materialized MLA attention feasibility (one layer) | Run `python -m --help` for arguments. Templates are listed in the [configuration index](../../configs/README.md). Supply your own model, initial @@ -75,6 +76,7 @@ completion are included. - [DeepSpec](deepspec.md): resident and native cache-first workflows. - [EAGLE3](eagle3-h200.md): synthetic train-core measurement and numerical scope. - [Packed attention/CP](cudnn-thd-cp.md): attention component results. +- [Absorbed MLA](absorbed-mla.md): absorbed versus materialized attention identity and a negative training-shape result. - [Training and acceptance](dspark-alignment.md): TorchSpec/ASpec BSHD/THD training alignment and verifier metrics. - [CP quality alignment](dspark-cp-alignment.md): CP2 versus same-layout CP1 diff --git a/docs/benchmarks/absorbed-mla.md b/docs/benchmarks/absorbed-mla.md new file mode 100644 index 0000000..37c96b5 --- /dev/null +++ b/docs/benchmarks/absorbed-mla.md @@ -0,0 +1,98 @@ +# Absorbed MLA feasibility (negative training result) + +This note is the feasibility milestone for [ASpec #19](https://github.com/inclusionAI/ASpec/issues/19). +It checks the absorbed-MLA identity on the DSpark draft geometry and measures whether +the naive absorbed form benefits draft training. It does **not** add a selectable +training path, a fused kernel or an export claim. + +## Derivation + +The retained materialized path computes, per draft layer, + +```text +k_nope = c_kv @ W_UK^T # [B, T, H, Dn] +v = c_kv @ W_UV^T # [B, T, H, Dv] +logits = q_nope @ k_nope^T + q_rope @ k_rope^T +out = softmax(logits) @ v +``` + +where `c_kv` is the post-norm latent KV (`kv_lora_rank`), `q_rope`/`k_rope` are the +RoPE side (`qk_rope_head_dim`) and `W_UK`/`W_UV` are the per-head blocks of +`kv_b_proj.weight`. Re-associating both products gives the absorbed form: + +```text +q_lat = q_nope @ W_UK # [B, S, H, K] +logits = q_lat @ c_kv^T + q_rope @ k_rope^T +out = (softmax(logits) @ c_kv) @ W_UV^T +``` + +RoPE still applies to the small side only, and both score terms are added before +the softmax. The identity is exact for the dual-source `cat(context, draft)` KV and +for `q_lora_rank` on/off: those projections happen before this boundary. + +## Scope of the prototype + +`aspec.models.ops.absorbed_mla.absorbed_mla_attention` implements the dense, +unmasked-or-boolean-mask case in PyTorch. `tests/cpu/test_absorbed_mla.py` checks: + +- forward parity against an explicitly materialized reference; +- gradients of both query sides, `c_kv`, `k_rope` and `kv_b_proj.weight`; +- dense masks (including broadcast masks that hide keys) and short sequences; +- parity with the retained `K3DSparkMLAAttention` projections and softmax scale. + +FlexAttention/cuDNN masks, packed THD layouts, Direct-CP, `o_proj` fusion and +numerical policies for a fused kernel are not covered. + +## Measurement + +`tools/performance/bench_absorbed_mla.py` measures forward+backward at the attention +boundary for one layer: BF16, `B=1`, `H=32`, `Dn=128`, `Dr=64`, `Dv=128`, +`K=512`, context 4096, block 7. It compares the retained fused-SDPA path, an +explicit-softmax materialized path (same logits materialization as absorbed) and +the absorbed path. No mask, `o_proj`, Target capture or optimizer step is included. + +Measured on 8x H100 PCIe 80GB, driver 580.126.09, PyTorch 2.13.0+cu130, BF16, +warmup 1, 3 runs, median. Peak memory is `torch.cuda.max_memory_allocated()` for +forward+backward. + +| anchors | `S` | `T` | materialized SDPA | materialized explicit | absorbed | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 7 | 4103 | 2.51 ms / 0.240 GiB | 3.27 ms / 0.184 GiB | 3.80 ms / 0.101 GiB | +| 8 | 56 | 4152 | 2.55 ms / 0.250 GiB | 3.23 ms / 0.260 GiB | 3.24 ms / 0.220 GiB | +| 128 | 896 | 4992 | 3.17 ms / 0.350 GiB | 9.20 ms / 2.32 GiB | 10.64 ms / 2.80 GiB | +| 512 | 3584 | 7680 | 9.46 ms / 0.670 GiB | 51.24 ms / 13.41 GiB | 63.46 ms / 16.69 GiB | + +`S` is `num_anchors * block_size`; `T = context_length + S`. + +Reproduce with: + +```bash +env -u LD_LIBRARY_PATH CUDA_VISIBLE_DEVICES=1 \ + python -m tools.performance.bench_absorbed_mla \ + --context-length 4096 --num-anchors 512 --block-size 7 \ + --warmup 1 --runs 3 --result /tmp/absorbed-mla.json +``` + +Keep the raw JSON outside the source tree. `LD_LIBRARY_PATH` was unset only to +select this PyTorch build's bundled cuDNN on the measurement host. + +## Result and recommendation + +For the default draft anchor budget (512 anchors, 3584 queries), the naive absorbed +path is both slower and larger than the retained fused-SDPA path: explicit logits +dominate memory, and `q_lat` has `kv_lora_rank=512` versus the materialized +`qk_nope_head_dim=128`, so the score matmul does more work. Only the tiny-query +regime (1 anchor, 7 queries) reduces peak activation memory (0.101 vs 0.240 GiB), +and even there it is slower. + +**Recommendation:** do not integrate the naive PyTorch absorbed form into maintained +draft training, and do not claim a training speedup. A useful follow-up would need +a fused absorbed kernel (score + softmax + latent output) or a workload where the +query/key ratio is genuinely tiny; both are separate from this prototype. The +identity and gradient checks remain useful for validating such a kernel. + +## Remaining work + +Fused forward/backward kernels, FlexAttention/cuDNN mask support, packing and +Direct-CP layouts, `o_proj` fusion, end-to-end memory/time including target feature +capture, exact resume and export. None of these are established by this note. diff --git a/tests/cpu/test_absorbed_mla.py b/tests/cpu/test_absorbed_mla.py new file mode 100644 index 0000000..75502d8 --- /dev/null +++ b/tests/cpu/test_absorbed_mla.py @@ -0,0 +1,243 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Absorbed-MLA feasibility checks. + +The reference expands ``kv_b_proj`` into per-head K/V and runs an explicit +materialized attention. These checks cover the forward identity, affected +gradients, dense masks and the DSpark MLA geometry; they do not establish a +production speedup. +""" + +import math + +import pytest +import torch + +from aspec.models.draft.dspark import ( + K3DSparkConfig, + K3DSparkMLAAttention, + _apply_rope_interleaved, +) +from aspec.models.ops.absorbed_mla import absorbed_mla_attention, split_kv_b_proj + + +def make_tensors(batch=1, query=4, key=7, heads=2, nope=4, rope=2, value=3, latent=5): + generator = torch.Generator().manual_seed(23) + return dict( + q_nope=torch.randn(batch, query, heads, nope, generator=generator), + q_rope=torch.randn(batch, query, heads, rope, generator=generator), + compressed_kv=torch.randn(batch, key, latent, generator=generator), + k_rope=torch.randn(batch, key, rope, generator=generator), + kv_b_proj_weight=torch.randn( + heads * (nope + value), latent, generator=generator + ), + ) + + +def reference_materialized( + q_nope, q_rope, compressed_kv, k_rope, kv_b_proj_weight, + *, softmax_scale=None, attn_mask=None, +): + """Explicitly materialize per-head K/V, then run dense attention.""" + batch, query, heads, nope = q_nope.shape + rope = q_rope.size(-1) + value = kv_b_proj_weight.shape[0] // heads - nope + latent = compressed_kv.size(-1) + view = kv_b_proj_weight.view(heads, nope + value, latent) + w_uk, w_uv = view[:, :nope, :], view[:, nope:, :] + k_nope = torch.einsum("btk,hnk->bthn", compressed_kv, w_uk) + values = torch.einsum("btk,hvk->bthv", compressed_kv, w_uv) + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(nope + rope) + scores = torch.einsum("bshn,bthn->bhst", q_nope, k_nope) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope, k_rope) + scores = scores * softmax_scale + if attn_mask is not None: + scores = scores.masked_fill(~attn_mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores.float(), dim=-1).to(q_nope.dtype) + output = torch.einsum("bhst,bthv->bshv", probs, values) + return output.reshape(batch, query, heads * value) + + +def make_attention_config(**overrides): + values = dict( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=2, vocab_size=32, + q_lora_rank=8, kv_lora_rank=5, qk_nope_head_dim=4, + qk_rope_head_dim=2, v_head_dim=3, num_target_layers=1, + target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=64, + rms_norm_eps=1e-6, rope_theta=10000.0, + ) + values.update(overrides) + return K3DSparkConfig(**values) + + +def project_q_and_latent(attention, draft_hidden, context_hidden, + draft_position_ids, context_position_ids): + """Reproduce the materialized MLA boundary before attention.""" + batch, draft_len, _ = draft_hidden.shape + query = attention.q_b_proj( + attention.q_a_layernorm(attention.q_a_proj(draft_hidden)) + ) + query = query.view(batch, draft_len, attention.num_heads, attention.qk_head_dim) + # The retained path transposes to [B, H, S, D] before RoPE. + query = query.transpose(1, 2) + q_nope, q_rope = torch.split( + query, [attention.qk_nope_head_dim, attention.qk_rope_head_dim], dim=-1 + ) + kv_input = torch.cat([context_hidden, draft_hidden], dim=1) + kv_combined = attention.kv_a_proj_with_mqa(kv_input) + compressed, k_rope = torch.split( + kv_combined, [attention.kv_lora_rank, attention.qk_rope_head_dim], dim=-1 + ) + compressed = attention.kv_a_layernorm(compressed) + k_rope = k_rope.unsqueeze(1) + positions = torch.cat([context_position_ids, draft_position_ids], dim=1) + cos, sin = attention.rotary_emb(q_rope, seq_len=positions.size(1)) + q_rope = _apply_rope_interleaved( + q_rope, cos, sin, draft_position_ids, unsqueeze_dim=1 + ) + k_rope = _apply_rope_interleaved( + k_rope, cos, sin, positions, unsqueeze_dim=1 + ) + # The materialized path keeps [B, H, T, D]; the absorbed reference takes + # the BSHD-latent boundary, so K rope is the shared [B, T, D] tensor. + return ( + q_nope.transpose(1, 2), + q_rope.transpose(1, 2), + compressed, + k_rope.squeeze(1), + ) + + +def test_absorbed_forward_matches_materialized_equations(): + tensors = make_tensors() + absorbed = absorbed_mla_attention(**tensors) + expected = reference_materialized(**tensors) + assert absorbed.shape == (1, 4, 6) + torch.testing.assert_close(absorbed, expected, rtol=2e-6, atol=2e-6) + + +@pytest.mark.parametrize("materialized_first", [True, False]) +def test_absorbed_gradients_match_materialized(materialized_first): + tensors = make_tensors(batch=2) + reference_tensors = {name: value.clone() for name, value in tensors.items()} + for value in tensors.values(): + value.requires_grad_(True) + for value in reference_tensors.values(): + value.requires_grad_(True) + + def run(fn, values): + output = fn(**values) + loss = output.float().pow(2).sum() + return torch.autograd.grad( + loss, [*values.values()], allow_unused=False + ) + + if materialized_first: + expected = run(reference_materialized, reference_tensors) + actual = run(absorbed_mla_attention, tensors) + else: + actual = run(absorbed_mla_attention, tensors) + expected = run(reference_materialized, reference_tensors) + for name, actual_grad, expected_grad in zip( + tensors, actual, expected, strict=True + ): + torch.testing.assert_close( + actual_grad, expected_grad, rtol=1e-5, atol=2e-5, msg=name + ) + + +def test_dense_mask_excludes_hidden_keys(): + tensors = make_tensors() + mask = torch.ones(1, 2, 4, 7, dtype=torch.bool) + mask[:, :, :, 5:] = False + expected = reference_materialized(**tensors, attn_mask=mask) + actual = absorbed_mla_attention(**tensors, attn_mask=mask) + torch.testing.assert_close(actual, expected, rtol=2e-6, atol=2e-6) + + changed = {name: value.clone() for name, value in tensors.items()} + changed["compressed_kv"][:, 5:] += 100.0 + changed["k_rope"][:, 5:] -= 100.0 + moved = absorbed_mla_attention(**changed, attn_mask=mask) + torch.testing.assert_close(moved, actual, rtol=0, atol=0) + + +def test_mask_broadcasts_over_batch_and_heads(): + tensors = make_tensors(batch=2) + mask = torch.tril(torch.ones(4, 7, dtype=torch.bool)) + actual = absorbed_mla_attention(**tensors, attn_mask=mask) + expected = reference_materialized(**tensors, attn_mask=mask) + torch.testing.assert_close(actual, expected, rtol=2e-6, atol=2e-6) + + +def test_short_and_single_token_sequences(): + tensors = make_tensors(query=1, key=1) + torch.testing.assert_close( + absorbed_mla_attention(**tensors), reference_materialized(**tensors), + rtol=2e-6, atol=2e-6, + ) + tensors = make_tensors(query=1, key=3) + torch.testing.assert_close( + absorbed_mla_attention(**tensors), reference_materialized(**tensors), + rtol=2e-6, atol=2e-6, + ) + + +def test_split_rejects_wrong_weight_shape(): + with pytest.raises(ValueError, match="kv_b_proj.weight"): + split_kv_b_proj( + torch.zeros(13, 5), num_heads=2, qk_nope_head_dim=4, + v_head_dim=3, kv_lora_rank=5, + ) + + +@pytest.mark.parametrize("mutation,match", [ + (lambda t: t.update(k_rope=t["k_rope"][:, :2]), "k_rope"), + (lambda t: t.update(k_rope=t["k_rope"].double()), "dtype"), + (lambda t: t.update(attn_mask=torch.ones(4, 7)), "boolean"), + (lambda t: t.update(attn_mask=torch.ones(1, 3, 4, 7, dtype=torch.bool)), "broadcast"), + (lambda t: t.update(softmax_scale=0.0), "softmax_scale"), + (lambda t: t.update(softmax_scale=float("nan")), "softmax_scale"), +]) +def test_invalid_inputs_fail_closed(mutation, match): + tensors = dict(make_tensors(), softmax_scale=None, attn_mask=None) + tensors.pop("softmax_scale") + tensors.pop("attn_mask") + mutation(tensors) + with pytest.raises(ValueError, match=match): + absorbed_mla_attention(**tensors) + + +def test_matches_the_retained_materialized_dspark_attention(): + torch.manual_seed(11) + attention = K3DSparkMLAAttention(make_attention_config()) + batch, query, context = 1, 3, 6 + draft_hidden = torch.randn(batch, query, 16) + context_hidden = torch.randn(batch, context, 16) + draft_positions = torch.arange(context, context + query).unsqueeze(0) + context_positions = torch.arange(context).unsqueeze(0) + + retained = attention( + draft_hidden, context_hidden, draft_positions, context_positions, + block_mask=None, + ) + q_nope, q_rope, compressed, k_rope = project_q_and_latent( + attention, draft_hidden, context_hidden, draft_positions, context_positions + ) + reference = reference_materialized( + q_nope, q_rope, compressed, k_rope, attention.kv_b_proj.weight, + softmax_scale=attention.softmax_scale, + ) + absorbed = absorbed_mla_attention( + q_nope, q_rope, compressed, k_rope, attention.kv_b_proj.weight, + softmax_scale=attention.softmax_scale, + ) + torch.testing.assert_close( + attention.o_proj(reference), retained, rtol=2e-6, atol=2e-6 + ) + torch.testing.assert_close( + attention.o_proj(absorbed), retained, rtol=2e-6, atol=2e-6 + ) diff --git a/tools/performance/bench_absorbed_mla.py b/tools/performance/bench_absorbed_mla.py new file mode 100644 index 0000000..26044c3 --- /dev/null +++ b/tools/performance/bench_absorbed_mla.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Feasibility benchmark for absorbed versus materialized MLA attention. + +The workload starts at the DSpark MLA attention boundary (BSHD query/latent +tensors, BF16, no dense mask). It compares: + +- ``materialized_sdpa``: expand ``kv_b_proj`` and call fused SDPA; +- ``materialized_explicit``: expand ``kv_b_proj`` and materialize logits; +- ``absorbed``: keep the normalized latent and re-associate both projections. + +FlexAttention/cuDNN masks, packed layouts and Direct-CP are outside this +benchmark. Report the boundary together with the result; this does not establish +end-to-end training throughput. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +import torch +import torch.nn.functional as F + +from aspec.models.ops.absorbed_mla import absorbed_mla_attention + + +@dataclass(frozen=True) +class Workload: + context_length: int = 4096 + num_anchors: int = 512 + block_size: int = 7 + num_heads: int = 32 + qk_nope_head_dim: int = 128 + qk_rope_head_dim: int = 64 + v_head_dim: int = 128 + kv_lora_rank: int = 512 + batch_size: int = 1 + + @property + def query_length(self) -> int: + return self.num_anchors * self.block_size + + @property + def key_length(self) -> int: + return self.context_length + self.query_length + + @property + def softmax_scale(self) -> float: + return 1.0 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim) + + +def build_inputs(workload: Workload, device: torch.device, dtype: torch.dtype): + generator = torch.Generator(device="cpu").manual_seed(7) + shape = (workload.batch_size, workload.query_length, workload.num_heads) + tensors = { + "q_nope": torch.randn( + *shape, workload.qk_nope_head_dim, generator=generator + ).to(device=device, dtype=dtype), + "q_rope": torch.randn( + *shape, workload.qk_rope_head_dim, generator=generator + ).to(device=device, dtype=dtype), + "compressed_kv": torch.randn( + workload.batch_size, workload.key_length, workload.kv_lora_rank, + generator=generator, + ).to(device=device, dtype=dtype), + "k_rope": torch.randn( + workload.batch_size, workload.key_length, workload.qk_rope_head_dim, + generator=generator, + ).to(device=device, dtype=dtype), + "kv_b_proj_weight": torch.randn( + workload.num_heads + * (workload.qk_nope_head_dim + workload.v_head_dim), + workload.kv_lora_rank, + generator=generator, + ).to(device=device, dtype=dtype), + } + for tensor in tensors.values(): + tensor.requires_grad_(True) + return tensors + + +def split_projection(weight, workload: Workload): + view = weight.view( + workload.num_heads, + workload.qk_nope_head_dim + workload.v_head_dim, + workload.kv_lora_rank, + ) + return view[:, : workload.qk_nope_head_dim], view[:, workload.qk_nope_head_dim :] + + +def materialized_kv(compressed_kv, weight, workload: Workload): + w_uk, w_uv = split_projection(weight, workload) + k_nope = torch.einsum("btk,hnk->bthn", compressed_kv, w_uk) + values = torch.einsum("btk,hvk->bthv", compressed_kv, w_uv) + return k_nope, values + + +def materialized_sdpa(tensors, workload: Workload): + q_nope = tensors["q_nope"].transpose(1, 2) + q_rope = tensors["q_rope"].transpose(1, 2) + k_nope, values = materialized_kv( + tensors["compressed_kv"], tensors["kv_b_proj_weight"], workload + ) + key = torch.cat( + [ + k_nope.transpose(1, 2), + tensors["k_rope"].unsqueeze(1).expand(-1, workload.num_heads, -1, -1), + ], + dim=-1, + ) + query = torch.cat([q_nope, q_rope], dim=-1) + output = F.scaled_dot_product_attention( + query, key, values.transpose(1, 2), is_causal=False, dropout_p=0.0, + scale=workload.softmax_scale, + ) + return output.transpose(1, 2).reshape( + workload.batch_size, workload.query_length, + workload.num_heads * workload.v_head_dim, + ) + + +def materialized_explicit(tensors, workload: Workload): + q_nope = tensors["q_nope"] + k_nope, values = materialized_kv( + tensors["compressed_kv"], tensors["kv_b_proj_weight"], workload + ) + scores = torch.einsum("bshn,bthn->bhst", q_nope, k_nope) + scores = scores + torch.einsum( + "bshr,btr->bhst", tensors["q_rope"], tensors["k_rope"] + ) + scores = scores * workload.softmax_scale + probs = torch.softmax(scores.float(), dim=-1).to(q_nope.dtype) + output = torch.einsum("bhst,bthv->bshv", probs, values) + return output.reshape( + workload.batch_size, workload.query_length, + workload.num_heads * workload.v_head_dim, + ) + + +def absorbed(tensors, workload: Workload): + return absorbed_mla_attention( + tensors["q_nope"], tensors["q_rope"], tensors["compressed_kv"], + tensors["k_rope"], tensors["kv_b_proj_weight"], + softmax_scale=workload.softmax_scale, + ) + + +def run_step(fn, tensors, workload): + output = fn(tensors, workload) + loss = output.float().pow(2).sum() + return torch.autograd.grad(loss, list(tensors.values())) + + +def measure(fn, tensors, workload, *, warmup: int, runs: int): + leaves = list(tensors.values()) + for _ in range(warmup): + run_step(fn, tensors, workload) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + samples = [] + for _ in range(runs): + torch.cuda.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + run_step(fn, tensors, workload) + end.record() + end.synchronize() + samples.append(start.elapsed_time(end)) + for leaf in leaves: + leaf.grad = None + return { + "median_ms": statistics.median(samples), + "min_ms": min(samples), + "max_ms": max(samples), + "samples_ms": samples, + "peak_memory_bytes": torch.cuda.max_memory_allocated(), + } + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--context-length", type=int, default=4096) + parser.add_argument("--num-anchors", type=int, default=512) + parser.add_argument("--block-size", type=int, default=7) + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16") + parser.add_argument("--result", type=Path) + args = parser.parse_args(argv) + if not torch.cuda.is_available(): + raise SystemExit("absorbed MLA benchmark requires CUDA") + if args.warmup < 1 or args.runs < 1: + raise SystemExit("warmup and runs must be positive") + workload = Workload( + context_length=args.context_length, + num_anchors=args.num_anchors, + block_size=args.block_size, + batch_size=args.batch_size, + ) + dtype = getattr(torch, args.dtype) + device = torch.device("cuda") + tensors = build_inputs(workload, device, dtype) + variants = { + "materialized_sdpa": materialized_sdpa, + "materialized_explicit": materialized_explicit, + "absorbed": absorbed, + } + results = { + "workload": asdict(workload) | { + "query_length": workload.query_length, + "key_length": workload.key_length, + "dtype": args.dtype, + }, + "device": torch.cuda.get_device_name(), + "torch": torch.__version__, + "boundary": "attention core, dense unmasked, forward+backward, no o_proj", + "variants": {}, + } + for name, fn in variants.items(): + results["variants"][name] = measure( + fn, tensors, workload, warmup=args.warmup, runs=args.runs + ) + print(name, json.dumps(results["variants"][name])) + if args.result: + args.result.write_text(json.dumps(results, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 27c5f5f0cfc8c9007dabf1a2e48b1ce9b2da85ec Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:20:24 +0800 Subject: [PATCH 2/4] fix(mla): fail closed on all-masked rows and cover the no-q-lora path - Reject empty visible-key rows instead of relying on softmax(-inf) NaN. - Reject boolean softmax scales. - Parameterize the retained-attention parity test over q_lora_rank on/off. --- aspec/models/ops/absorbed_mla.py | 13 +++++++++++-- tests/cpu/test_absorbed_mla.py | 25 ++++++++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/aspec/models/ops/absorbed_mla.py b/aspec/models/ops/absorbed_mla.py index c1e5686..dec0cc9 100644 --- a/aspec/models/ops/absorbed_mla.py +++ b/aspec/models/ops/absorbed_mla.py @@ -125,7 +125,11 @@ def absorbed_mla_attention( raise ValueError("absorbed MLA requires floating-point tensors") if softmax_scale is None: softmax_scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim) - if not math.isfinite(softmax_scale) or softmax_scale <= 0: + if ( + isinstance(softmax_scale, bool) + or not math.isfinite(softmax_scale) + or softmax_scale <= 0 + ): raise ValueError("softmax_scale must be positive and finite") q_latent = torch.einsum("bshn,hnk->bshk", q_nope, w_uk) @@ -136,11 +140,16 @@ def absorbed_mla_attention( if attn_mask.dtype != torch.bool: raise ValueError("attn_mask must be boolean") try: - scores = scores.masked_fill(~attn_mask, torch.finfo(scores.dtype).min) + mask = attn_mask.expand(scores.shape) except RuntimeError as exc: raise ValueError( "attn_mask must broadcast against [B, H, S, T]" ) from exc + if not mask.any(dim=-1).all(): + raise ValueError( + "attn_mask must leave at least one visible key per query" + ) + scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) probs = torch.softmax(scores.float(), dim=-1).to(dtype) out_latent = torch.einsum("bhst,btk->bshk", probs, compressed_kv) output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) diff --git a/tests/cpu/test_absorbed_mla.py b/tests/cpu/test_absorbed_mla.py index 75502d8..0f65a0e 100644 --- a/tests/cpu/test_absorbed_mla.py +++ b/tests/cpu/test_absorbed_mla.py @@ -78,9 +78,12 @@ def project_q_and_latent(attention, draft_hidden, context_hidden, draft_position_ids, context_position_ids): """Reproduce the materialized MLA boundary before attention.""" batch, draft_len, _ = draft_hidden.shape - query = attention.q_b_proj( - attention.q_a_layernorm(attention.q_a_proj(draft_hidden)) - ) + if attention.q_lora_rank is not None: + query = attention.q_b_proj( + attention.q_a_layernorm(attention.q_a_proj(draft_hidden)) + ) + else: + query = attention.q_proj(draft_hidden) query = query.view(batch, draft_len, attention.num_heads, attention.qk_head_dim) # The retained path transposes to [B, H, S, D] before RoPE. query = query.transpose(1, 2) @@ -165,6 +168,14 @@ def test_dense_mask_excludes_hidden_keys(): torch.testing.assert_close(moved, actual, rtol=0, atol=0) +def test_fully_masked_rows_fail_closed(): + tensors = make_tensors() + mask = torch.ones(1, 2, 4, 7, dtype=torch.bool) + mask[:, :, :2] = False + with pytest.raises(ValueError, match="visible key"): + absorbed_mla_attention(**tensors, attn_mask=mask) + + def test_mask_broadcasts_over_batch_and_heads(): tensors = make_tensors(batch=2) mask = torch.tril(torch.ones(4, 7, dtype=torch.bool)) @@ -201,6 +212,7 @@ def test_split_rejects_wrong_weight_shape(): (lambda t: t.update(attn_mask=torch.ones(1, 3, 4, 7, dtype=torch.bool)), "broadcast"), (lambda t: t.update(softmax_scale=0.0), "softmax_scale"), (lambda t: t.update(softmax_scale=float("nan")), "softmax_scale"), + (lambda t: t.update(softmax_scale=True), "softmax_scale"), ]) def test_invalid_inputs_fail_closed(mutation, match): tensors = dict(make_tensors(), softmax_scale=None, attn_mask=None) @@ -211,9 +223,12 @@ def test_invalid_inputs_fail_closed(mutation, match): absorbed_mla_attention(**tensors) -def test_matches_the_retained_materialized_dspark_attention(): +@pytest.mark.parametrize("q_lora_rank", [8, None]) +def test_matches_the_retained_materialized_dspark_attention(q_lora_rank): torch.manual_seed(11) - attention = K3DSparkMLAAttention(make_attention_config()) + attention = K3DSparkMLAAttention( + make_attention_config(q_lora_rank=q_lora_rank) + ) batch, query, context = 1, 3, 6 draft_hidden = torch.randn(batch, query, 16) context_hidden = torch.randn(batch, context, 16) From 4a8da36b5ba53b917f1a51d6a689b6934d6a71a7 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:25:28 +0800 Subject: [PATCH 3/4] fix(mla): zero empty-mask rows and counterbalance benchmark order - Empty mask rows now produce zero output/gradient like fused SDPA instead of failing closed; cover it against an independent reference. - Rotate variant order per measurement round and record order plus per-round timings/peaks. - Refresh the measured table and include the eight-anchor memory case. --- aspec/models/ops/absorbed_mla.py | 11 ++-- docs/benchmarks/absorbed-mla.md | 23 ++++---- tests/cpu/test_absorbed_mla.py | 21 ++++++-- tools/performance/bench_absorbed_mla.py | 72 +++++++++++++++---------- 4 files changed, 78 insertions(+), 49 deletions(-) diff --git a/aspec/models/ops/absorbed_mla.py b/aspec/models/ops/absorbed_mla.py index dec0cc9..b7cd70b 100644 --- a/aspec/models/ops/absorbed_mla.py +++ b/aspec/models/ops/absorbed_mla.py @@ -145,12 +145,13 @@ def absorbed_mla_attention( raise ValueError( "attn_mask must broadcast against [B, H, S, T]" ) from exc - if not mask.any(dim=-1).all(): - raise ValueError( - "attn_mask must leave at least one visible key per query" - ) scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) - probs = torch.softmax(scores.float(), dim=-1).to(dtype) + probs = torch.softmax(scores.float(), dim=-1) + if attn_mask is not None: + # Match fused SDPA: rows with no visible key contribute zero, not a + # uniform average over the finite masked scores. + probs = probs * mask.any(dim=-1, keepdim=True) + probs = probs.to(dtype) out_latent = torch.einsum("bhst,btk->bshk", probs, compressed_kv) output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) return output.reshape(batch, query_len, num_heads * inferred_v_head_dim) diff --git a/docs/benchmarks/absorbed-mla.md b/docs/benchmarks/absorbed-mla.md index 37c96b5..f91547c 100644 --- a/docs/benchmarks/absorbed-mla.md +++ b/docs/benchmarks/absorbed-mla.md @@ -33,11 +33,13 @@ for `q_lora_rank` on/off: those projections happen before this boundary. ## Scope of the prototype `aspec.models.ops.absorbed_mla.absorbed_mla_attention` implements the dense, -unmasked-or-boolean-mask case in PyTorch. `tests/cpu/test_absorbed_mla.py` checks: +unmasked-or-boolean-mask case in PyTorch. Empty mask rows produce zero output and +zero gradient, matching fused SDPA. `tests/cpu/test_absorbed_mla.py` checks: - forward parity against an explicitly materialized reference; - gradients of both query sides, `c_kv`, `k_rope` and `kv_b_proj.weight`; -- dense masks (including broadcast masks that hide keys) and short sequences; +- dense masks (including broadcast masks that hide keys and empty rows) and + short sequences; - parity with the retained `K3DSparkMLAAttention` projections and softmax scale. FlexAttention/cuDNN masks, packed THD layouts, Direct-CP, `o_proj` fusion and @@ -53,14 +55,15 @@ the absorbed path. No mask, `o_proj`, Target capture or optimizer step is includ Measured on 8x H100 PCIe 80GB, driver 580.126.09, PyTorch 2.13.0+cu130, BF16, warmup 1, 3 runs, median. Peak memory is `torch.cuda.max_memory_allocated()` for -forward+backward. +forward+backward. Variants run in rotating order per round; the tool records each +round's order and the per-round timings and peaks in the JSON result. | anchors | `S` | `T` | materialized SDPA | materialized explicit | absorbed | | ---: | ---: | ---: | ---: | ---: | ---: | -| 1 | 7 | 4103 | 2.51 ms / 0.240 GiB | 3.27 ms / 0.184 GiB | 3.80 ms / 0.101 GiB | -| 8 | 56 | 4152 | 2.55 ms / 0.250 GiB | 3.23 ms / 0.260 GiB | 3.24 ms / 0.220 GiB | -| 128 | 896 | 4992 | 3.17 ms / 0.350 GiB | 9.20 ms / 2.32 GiB | 10.64 ms / 2.80 GiB | -| 512 | 3584 | 7680 | 9.46 ms / 0.670 GiB | 51.24 ms / 13.41 GiB | 63.46 ms / 16.69 GiB | +| 1 | 7 | 4103 | 3.38 ms / 0.240 GiB | 4.02 ms / 0.184 GiB | 4.32 ms / 0.101 GiB | +| 8 | 56 | 4152 | 3.26 ms / 0.245 GiB | 3.60 ms / 0.258 GiB | 3.91 ms / 0.225 GiB | +| 128 | 896 | 4992 | 3.02 ms / 0.348 GiB | 9.04 ms / 2.315 GiB | 10.49 ms / 2.799 GiB | +| 512 | 3584 | 7680 | 9.14 ms / 0.668 GiB | 51.50 ms / 13.411 GiB | 63.11 ms / 16.688 GiB | `S` is `num_anchors * block_size`; `T = context_length + S`. @@ -81,9 +84,9 @@ select this PyTorch build's bundled cuDNN on the measurement host. For the default draft anchor budget (512 anchors, 3584 queries), the naive absorbed path is both slower and larger than the retained fused-SDPA path: explicit logits dominate memory, and `q_lat` has `kv_lora_rank=512` versus the materialized -`qk_nope_head_dim=128`, so the score matmul does more work. Only the tiny-query -regime (1 anchor, 7 queries) reduces peak activation memory (0.101 vs 0.240 GiB), -and even there it is slower. +`qk_nope_head_dim=128`, so the score matmul does more work. The tiny-query +regimes (1 or 8 anchors) do reduce peak activation memory (0.101 vs 0.240 GiB at +one anchor, 0.225 vs 0.245 GiB at eight), but both are slower. **Recommendation:** do not integrate the naive PyTorch absorbed form into maintained draft training, and do not claim a training speedup. A useful follow-up would need diff --git a/tests/cpu/test_absorbed_mla.py b/tests/cpu/test_absorbed_mla.py index 0f65a0e..1e3dd2f 100644 --- a/tests/cpu/test_absorbed_mla.py +++ b/tests/cpu/test_absorbed_mla.py @@ -54,8 +54,12 @@ def reference_materialized( scores = scores + torch.einsum("bshr,btr->bhst", q_rope, k_rope) scores = scores * softmax_scale if attn_mask is not None: - scores = scores.masked_fill(~attn_mask, torch.finfo(scores.dtype).min) - probs = torch.softmax(scores.float(), dim=-1).to(q_nope.dtype) + mask = attn_mask.expand(scores.shape) + scores = scores.masked_fill(~mask, torch.finfo(scores.dtype).min) + probs = torch.softmax(scores.float(), dim=-1) + if attn_mask is not None: + probs = probs * mask.any(dim=-1, keepdim=True) + probs = probs.to(q_nope.dtype) output = torch.einsum("bhst,bthv->bshv", probs, values) return output.reshape(batch, query, heads * value) @@ -168,12 +172,19 @@ def test_dense_mask_excludes_hidden_keys(): torch.testing.assert_close(moved, actual, rtol=0, atol=0) -def test_fully_masked_rows_fail_closed(): +def test_fully_masked_rows_have_zero_output_and_gradient(): tensors = make_tensors() + for value in tensors.values(): + value.requires_grad_(True) mask = torch.ones(1, 2, 4, 7, dtype=torch.bool) mask[:, :, :2] = False - with pytest.raises(ValueError, match="visible key"): - absorbed_mla_attention(**tensors, attn_mask=mask) + output = absorbed_mla_attention(**tensors, attn_mask=mask) + assert torch.count_nonzero(output[:, :2]) == 0 + grads = torch.autograd.grad(output.float().pow(2).sum(), list(tensors.values())) + q_nope_grad = grads[0] + assert torch.count_nonzero(q_nope_grad[:, :2]) == 0 + expected = reference_materialized(**tensors, attn_mask=mask) + torch.testing.assert_close(output, expected, rtol=2e-6, atol=2e-6) def test_mask_broadcasts_over_batch_and_heads(): diff --git a/tools/performance/bench_absorbed_mla.py b/tools/performance/bench_absorbed_mla.py index 26044c3..2f6de1b 100644 --- a/tools/performance/bench_absorbed_mla.py +++ b/tools/performance/bench_absorbed_mla.py @@ -170,31 +170,43 @@ def run_step(fn, tensors, workload): return torch.autograd.grad(loss, list(tensors.values())) -def measure(fn, tensors, workload, *, warmup: int, runs: int): - leaves = list(tensors.values()) - for _ in range(warmup): - run_step(fn, tensors, workload) +def measure_variants(variants, tensors, workload, *, warmup: int, runs: int): + """Time each variant in rotating order and record per-round raw samples.""" + names = list(variants) + for round_index in range(warmup): + order = names[round_index % len(names):] + names[: round_index % len(names)] + for name in order: + run_step(variants[name], tensors, workload) torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - samples = [] - for _ in range(runs): - torch.cuda.synchronize() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - run_step(fn, tensors, workload) - end.record() - end.synchronize() - samples.append(start.elapsed_time(end)) - for leaf in leaves: - leaf.grad = None - return { - "median_ms": statistics.median(samples), - "min_ms": min(samples), - "max_ms": max(samples), - "samples_ms": samples, - "peak_memory_bytes": torch.cuda.max_memory_allocated(), - } + samples = {name: [] for name in names} + peaks = {name: [] for name in names} + orders = [] + for round_index in range(runs): + offset = round_index % len(names) + order = names[offset:] + names[:offset] + orders.append(order) + for name in order: + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + run_step(variants[name], tensors, workload) + end.record() + end.synchronize() + samples[name].append(start.elapsed_time(end)) + peaks[name].append(torch.cuda.max_memory_allocated()) + results = {} + for name in names: + results[name] = { + "median_ms": statistics.median(samples[name]), + "min_ms": min(samples[name]), + "max_ms": max(samples[name]), + "samples_ms": samples[name], + "peak_memory_samples_bytes": peaks[name], + "peak_memory_bytes": max(peaks[name]), + } + return results, orders def main(argv=None) -> int: @@ -237,11 +249,13 @@ def main(argv=None) -> int: "boundary": "attention core, dense unmasked, forward+backward, no o_proj", "variants": {}, } - for name, fn in variants.items(): - results["variants"][name] = measure( - fn, tensors, workload, warmup=args.warmup, runs=args.runs - ) - print(name, json.dumps(results["variants"][name])) + measured, orders = measure_variants( + variants, tensors, workload, warmup=args.warmup, runs=args.runs + ) + results["variant_order"] = orders + results["variants"] = measured + for name, value in measured.items(): + print(name, json.dumps(value)) if args.result: args.result.write_text(json.dumps(results, indent=2) + "\n") return 0 From 898157b4903cc9a9108d1b01ab1da870e8bd2f33 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:08:31 +0800 Subject: [PATCH 4/4] feat(mla): add fused Triton absorbed-attention kernel Add a FlashAttention-style Triton forward and deterministic grouped backward for the absorbed-MLA core (score + online softmax + latent output), plus the q_lat/output projections around it. The key/value-side backward reduces heads through per-group FP32 partials instead of atomics or a serial all-heads loop. Add GPU parity/gradient tests against the dense reference, a fused variant in the absorbed-MLA benchmark, and a follow-up benchmark note. Dense masks, packed layouts and o_proj fusion remain out of scope. --- aspec/models/ops/absorbed_mla_fused.py | 796 ++++++++++++++++++++++++ docs/benchmarks/README.md | 3 +- docs/benchmarks/absorbed-mla-fused.md | 95 +++ tests/test_absorbed_mla_fused_gpu.py | 148 +++++ tools/performance/bench_absorbed_mla.py | 18 +- 5 files changed, 1058 insertions(+), 2 deletions(-) create mode 100644 aspec/models/ops/absorbed_mla_fused.py create mode 100644 docs/benchmarks/absorbed-mla-fused.md create mode 100644 tests/test_absorbed_mla_fused_gpu.py diff --git a/aspec/models/ops/absorbed_mla_fused.py b/aspec/models/ops/absorbed_mla_fused.py new file mode 100644 index 0000000..2d063f2 --- /dev/null +++ b/aspec/models/ops/absorbed_mla_fused.py @@ -0,0 +1,796 @@ +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fused absorbed-MLA attention with a FlashAttention-style Triton kernel. + +The reference implementation in :mod:`aspec.models.ops.absorbed_mla` keeps the +logits dense and materializes them with ``einsum``. This module keeps the same +absorbed identity but fuses the score, online softmax and latent-output +accumulation, so it never materializes ``[B, H, S, T]`` scores or per-head +``k_nope``/``v`` tensors: + + q_lat = q_nope @ W_UK # [B, S, H, K] + logits = q_lat @ c_kv^T + q_rope @ k_rope^T + out = softmax(logits) @ c_kv # [B, S, H, K] + output = out @ W_UV^T # [B, S, H * Dv] + +``W_UK``/``W_UV`` are per-head blocks of ``kv_b_proj.weight``; the latent +projection before and after the attention core stay in PyTorch so the gradient +for ``q_nope``/``kv_b_proj.weight`` flows through normal autograd. + +This is a feasibility kernel, not a production path: it supports dense, +unmasked attention on contiguous BSHD/BTD tensors, and dense masks remain on the +reference implementation. Backward uses two deterministic kernels (query-side +and key/value-side) and never needs atomics. +""" + +from __future__ import annotations + +import math + +import torch + +from aspec.models.ops.absorbed_mla import split_kv_b_proj + +try: # pragma: no cover - import guard for CPU-only installs + import triton + import triton.language as tl + + _HAS_TRITON = True +except ImportError: # pragma: no cover + triton = None + tl = None + _HAS_TRITON = False + + +def _next_power_of_2(value: int) -> int: + return 1 << max(0, int(value) - 1).bit_length() + + +# The latent output forces a [BLOCK_N, BLOCK_K] FP32 accumulator in the +# key/value-side backward, so the key tile stays smaller than the query tile. +# ``_DKDV_GROUP`` bounds the head-reduction workspace: each program owns +# ``num_heads / GROUP`` partial latent-gradient tensors instead of one per head. +_FWD_TILE_M = 32 +_FWD_TILE_N = 64 +_FWD_WARPS = 4 +_FWD_STAGES = 2 +_DQ_TILE_M = 16 +_DQ_TILE_N = 64 +_DQ_WARPS = 4 +_DQ_STAGES = 2 +_DKDV_TILE_M = 64 +_DKDV_TILE_N = 32 +_DKDV_WARPS = 4 +_DKDV_STAGES = 2 +_DKDV_GROUP = 8 + + +def _tile_sizes( + key_dim: int, + rope_dim: int, + block_m: int, + block_n: int, + query_len: int, +) -> tuple[int, int, int, int]: + block_k = max(16, _next_power_of_2(key_dim)) + block_dr = max(16, _next_power_of_2(rope_dim)) + # A long configured query tile is wasted on short draft blocks. + block_m = max(16, min(block_m, _next_power_of_2(query_len))) + return block_m, block_n, block_k, block_dr + + +def _dkdv_group(num_heads: int, preferred: int | None = None) -> int: + if preferred is None: + preferred = _DKDV_GROUP + group = max(1, min(int(preferred), int(num_heads))) + while num_heads % group: + group -= 1 + return group + + +if _HAS_TRITON: + + @triton.jit + def _absorbed_mla_fwd_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + out_ptr, + lse_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_o_b, + stride_o_s, + stride_o_h, + stride_l_b, + stride_l_h, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + ): + pid_m = tl.program_id(0) + pid_bh = tl.program_id(1) + b = pid_bh // H + h = pid_bh % H + + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_m = offs_m < S + mask_k = offs_k < K + mask_dr = offs_dr < DR + + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + + m_i = tl.full([BLOCK_M], float("-inf"), tl.float32) + l_i = tl.zeros([BLOCK_M], tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_K], tl.float32) + + for start_n in range(0, T, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + mask_n = offs_n < T + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + + m_new = tl.maximum(m_i, tl.max(scores, 1)) + alpha = tl.exp(m_i - m_new) + probs = tl.exp(scores - m_new[:, None]) + l_i = l_i * alpha + tl.sum(probs, 1) + acc = acc * alpha[:, None] + tl.dot(probs.to(c.dtype), c) + m_i = m_new + + l_safe = tl.where(l_i > 0, l_i, 1.0) + acc = acc / l_safe[:, None] + lse = m_i + tl.log(l_safe) + tl.store( + out_ptr + + b * stride_o_b + + offs_m[:, None] * stride_o_s + + h * stride_o_h + + offs_k[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_k[None, :], + ) + tl.store(lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, lse, mask=mask_m) + + @triton.jit + def _absorbed_mla_bwd_dq_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + dout_ptr, + lse_ptr, + d_ptr, + dq_lat_ptr, + dq_rope_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_do_b, + stride_do_s, + stride_do_h, + stride_l_b, + stride_l_h, + stride_d_b, + stride_d_s, + stride_d_h, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + ): + pid_m = tl.program_id(0) + pid_bh = tl.program_id(1) + b = pid_bh // H + h = pid_bh % H + + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_m = offs_m < S + mask_k = offs_k < K + mask_dr = offs_dr < DR + + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + dout = tl.load( + dout_ptr + + b * stride_do_b + + offs_m[:, None] * stride_do_s + + h * stride_do_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + lse = tl.load( + lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, + mask=mask_m, + other=0.0, + ) + d_i = tl.load( + d_ptr + b * stride_d_b + offs_m * stride_d_s + h * stride_d_h, + mask=mask_m, + other=0.0, + ) + + dq_lat = tl.zeros([BLOCK_M, BLOCK_K], tl.float32) + dq_rope = tl.zeros([BLOCK_M, BLOCK_DR], tl.float32) + + for start_n in range(0, T, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + mask_n = offs_n < T + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + probs = tl.exp(scores - lse[:, None]) + probs = tl.where(mask_m[:, None] & mask_n[None, :], probs, 0.0) + + dp = tl.dot(dout, tl.trans(c)) + ds = probs * (dp - d_i[:, None]) * scale + dq_lat += tl.dot(ds.to(c.dtype), c) + dq_rope += tl.dot(ds.to(kr.dtype), kr) + + tl.store( + dq_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + dq_lat.to(dq_lat_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_k[None, :], + ) + tl.store( + dq_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + dq_rope.to(dq_rope_ptr.dtype.element_ty), + mask=mask_m[:, None] & mask_dr[None, :], + ) + + @triton.jit + def _absorbed_mla_bwd_dkdv_kernel( + q_lat_ptr, + q_rope_ptr, + c_kv_ptr, + k_rope_ptr, + dout_ptr, + lse_ptr, + d_ptr, + dc_part_ptr, + dkr_part_ptr, + S, + T, + H, + K, + DR, + scale, + stride_ql_b, + stride_ql_s, + stride_ql_h, + stride_qr_b, + stride_qr_s, + stride_qr_h, + stride_c_b, + stride_c_t, + stride_kr_b, + stride_kr_t, + stride_do_b, + stride_do_s, + stride_do_h, + stride_l_b, + stride_l_h, + stride_d_b, + stride_d_s, + stride_d_h, + stride_dc_g, + stride_dc_b, + stride_dc_t, + stride_dk_g, + stride_dk_b, + stride_dk_t, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_DR: tl.constexpr, + GROUP: tl.constexpr, + ): + pid_n = tl.program_id(0) + b = tl.program_id(1) + pid_g = tl.program_id(2) + head0 = pid_g * GROUP + + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + offs_dr = tl.arange(0, BLOCK_DR) + mask_n = offs_n < T + mask_k = offs_k < K + mask_dr = offs_dr < DR + + c = tl.load( + c_kv_ptr + + b * stride_c_b + + offs_n[:, None] * stride_c_t + + offs_k[None, :], + mask=mask_n[:, None] & mask_k[None, :], + other=0.0, + ) + kr = tl.load( + k_rope_ptr + + b * stride_kr_b + + offs_n[:, None] * stride_kr_t + + offs_dr[None, :], + mask=mask_n[:, None] & mask_dr[None, :], + other=0.0, + ) + dc = tl.zeros([BLOCK_N, BLOCK_K], tl.float32) + dkr = tl.zeros([BLOCK_N, BLOCK_DR], tl.float32) + + # Each program owns a disjoint head group, so the head reduction is + # written to a per-group workspace instead of racing through atomics. + for hi in range(0, GROUP): + h = head0 + hi + for start_m in range(0, S, BLOCK_M): + offs_m = start_m + tl.arange(0, BLOCK_M) + mask_m = offs_m < S + q_lat = tl.load( + q_lat_ptr + + b * stride_ql_b + + offs_m[:, None] * stride_ql_s + + h * stride_ql_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + q_rope = tl.load( + q_rope_ptr + + b * stride_qr_b + + offs_m[:, None] * stride_qr_s + + h * stride_qr_h + + offs_dr[None, :], + mask=mask_m[:, None] & mask_dr[None, :], + other=0.0, + ) + dout = tl.load( + dout_ptr + + b * stride_do_b + + offs_m[:, None] * stride_do_s + + h * stride_do_h + + offs_k[None, :], + mask=mask_m[:, None] & mask_k[None, :], + other=0.0, + ) + lse = tl.load( + lse_ptr + b * stride_l_b + h * stride_l_h + offs_m, + mask=mask_m, + other=0.0, + ) + d_i = tl.load( + d_ptr + b * stride_d_b + offs_m * stride_d_s + h * stride_d_h, + mask=mask_m, + other=0.0, + ) + + scores = tl.dot(q_lat, tl.trans(c)) + scores += tl.dot(q_rope, tl.trans(kr)) + scores = scores * scale + scores = tl.where(mask_n[None, :], scores, -1.0e30) + probs = tl.exp(scores - lse[:, None]) + probs = tl.where(mask_m[:, None] & mask_n[None, :], probs, 0.0) + + dp = tl.dot(dout, tl.trans(c)) + ds = probs * (dp - d_i[:, None]) * scale + dc += tl.dot(tl.trans(ds.to(c.dtype)), q_lat) + dc += tl.dot(tl.trans(probs.to(c.dtype)), dout) + dkr += tl.dot(tl.trans(ds.to(kr.dtype)), q_rope) + + base_dc = dc_part_ptr + pid_g * stride_dc_g + b * stride_dc_b + base_dk = dkr_part_ptr + pid_g * stride_dk_g + b * stride_dk_b + tl.store( + base_dc + offs_n[:, None] * stride_dc_t + offs_k[None, :], + dc, + mask=mask_n[:, None] & mask_k[None, :], + ) + tl.store( + base_dk + offs_n[:, None] * stride_dk_t + offs_dr[None, :], + dkr, + mask=mask_n[:, None] & mask_dr[None, :], + ) + + +def _require_fused_support(tensor: torch.Tensor) -> None: + if not _HAS_TRITON: + raise RuntimeError("fused absorbed MLA requires triton") + if not tensor.is_cuda: + raise RuntimeError("fused absorbed MLA requires CUDA tensors") + if tensor.dtype not in (torch.bfloat16, torch.float16): + raise RuntimeError( + "fused absorbed MLA supports bfloat16/float16, got " + f"{tensor.dtype}" + ) + + +class _AbsorbedMLAFusedCore(torch.autograd.Function): + @staticmethod + def forward(ctx, q_lat, q_rope, compressed_kv, k_rope, softmax_scale): + _require_fused_support(q_lat) + for name, tensor in ( + ("q_rope", q_rope), + ("compressed_kv", compressed_kv), + ("k_rope", k_rope), + ): + if tensor.dtype != q_lat.dtype: + raise ValueError(f"{name} dtype must match q_lat") + if q_lat.ndim != 4 or q_rope.ndim != 4 or compressed_kv.ndim != 3: + raise ValueError("fused absorbed MLA expects [B,S,H,K], [B,S,H,Dr], [B,T,K]") + if q_rope.shape[:3] != q_lat.shape[:3]: + raise ValueError("q_rope must match q_lat batch/query/head axes") + if k_rope.shape != (compressed_kv.size(0), compressed_kv.size(1), q_rope.size(-1)): + raise ValueError("k_rope must be [B,T,Dr]") + + q_lat = q_lat.contiguous() + q_rope = q_rope.contiguous() + compressed_kv = compressed_kv.contiguous() + k_rope = k_rope.contiguous() + batch, query_len, num_heads, key_dim = q_lat.shape + key_len = compressed_kv.size(1) + rope_dim = q_rope.size(-1) + + output = torch.empty_like(q_lat) + lse = torch.empty( + (batch, num_heads, query_len), + device=q_lat.device, + dtype=torch.float32, + ) + block_m, block_n, block_k, block_dr = _tile_sizes( + key_dim, rope_dim, _FWD_TILE_M, _FWD_TILE_N, query_len + ) + grid = (triton.cdiv(query_len, block_m), batch * num_heads) + _absorbed_mla_fwd_kernel[grid]( + q_lat, + q_rope, + compressed_kv, + k_rope, + output, + lse, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + softmax_scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + BLOCK_M=block_m, + BLOCK_N=block_n, + BLOCK_K=block_k, + BLOCK_DR=block_dr, + num_warps=_FWD_WARPS, + num_stages=_FWD_STAGES, + ) + ctx.save_for_backward(q_lat, q_rope, compressed_kv, k_rope, output, lse) + ctx.softmax_scale = float(softmax_scale) + return output + + @staticmethod + def backward(ctx, grad_output): + q_lat, q_rope, compressed_kv, k_rope, output, lse = ctx.saved_tensors + grad_output = grad_output.contiguous() + batch, query_len, num_heads, key_dim = q_lat.shape + key_len = compressed_kv.size(1) + rope_dim = q_rope.size(-1) + scale = ctx.softmax_scale + + d_i = (grad_output.float() * output.float()).sum(-1) + grad_q_lat = torch.empty_like(q_lat) + grad_q_rope = torch.empty_like(q_rope) + dq_m, dq_n, dq_k, dq_dr = _tile_sizes( + key_dim, rope_dim, _DQ_TILE_M, _DQ_TILE_N, query_len + ) + + grid_m = (triton.cdiv(query_len, dq_m), batch * num_heads) + _absorbed_mla_bwd_dq_kernel[grid_m]( + q_lat, + q_rope, + compressed_kv, + k_rope, + grad_output, + lse, + d_i, + grad_q_lat, + grad_q_rope, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + grad_output.stride(0), + grad_output.stride(1), + grad_output.stride(2), + lse.stride(0), + lse.stride(1), + d_i.stride(0), + d_i.stride(1), + d_i.stride(2), + BLOCK_M=dq_m, + BLOCK_N=dq_n, + BLOCK_K=dq_k, + BLOCK_DR=dq_dr, + num_warps=_DQ_WARPS, + num_stages=_DQ_STAGES, + ) + + dk_m, dk_n, dk_k, dk_dr = _tile_sizes( + key_dim, rope_dim, _DKDV_TILE_M, _DKDV_TILE_N, query_len + ) + group = _dkdv_group(num_heads) + num_groups = num_heads // group + dc_part = torch.zeros( + (num_groups, batch, key_len, key_dim), + device=q_lat.device, + dtype=torch.float32, + ) + dkr_part = torch.zeros( + (num_groups, batch, key_len, rope_dim), + device=q_lat.device, + dtype=torch.float32, + ) + grid_n = (triton.cdiv(key_len, dk_n), batch, num_groups) + _absorbed_mla_bwd_dkdv_kernel[grid_n]( + q_lat, + q_rope, + compressed_kv, + k_rope, + grad_output, + lse, + d_i, + dc_part, + dkr_part, + query_len, + key_len, + num_heads, + key_dim, + rope_dim, + scale, + q_lat.stride(0), + q_lat.stride(1), + q_lat.stride(2), + q_rope.stride(0), + q_rope.stride(1), + q_rope.stride(2), + compressed_kv.stride(0), + compressed_kv.stride(1), + k_rope.stride(0), + k_rope.stride(1), + grad_output.stride(0), + grad_output.stride(1), + grad_output.stride(2), + lse.stride(0), + lse.stride(1), + d_i.stride(0), + d_i.stride(1), + d_i.stride(2), + dc_part.stride(0), + dc_part.stride(1), + dc_part.stride(2), + dkr_part.stride(0), + dkr_part.stride(1), + dkr_part.stride(2), + BLOCK_M=dk_m, + BLOCK_N=dk_n, + BLOCK_K=dk_k, + BLOCK_DR=dk_dr, + GROUP=group, + num_warps=_DKDV_WARPS, + num_stages=_DKDV_STAGES, + ) + grad_c_kv = dc_part.sum(dim=0).to(compressed_kv.dtype) + grad_k_rope = dkr_part.sum(dim=0).to(k_rope.dtype) + return grad_q_lat, grad_q_rope, grad_c_kv, grad_k_rope, None + + +def fused_absorbed_mla_core( + q_lat: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + *, + softmax_scale: float, +) -> torch.Tensor: + """Fused absorbed attention core: ``[B,S,H,K] -> [B,S,H,K]``. + + ``softmax_scale`` is required because the core cannot recover + ``qk_nope_head_dim`` from the latent width alone. + """ + return _AbsorbedMLAFusedCore.apply( + q_lat, q_rope, compressed_kv, k_rope, float(softmax_scale) + ) + + +def fused_absorbed_mla_attention( + q_nope: torch.Tensor, + q_rope: torch.Tensor, + compressed_kv: torch.Tensor, + k_rope: torch.Tensor, + kv_b_proj_weight: torch.Tensor, + *, + softmax_scale: float | None = None, + attn_mask: torch.Tensor | None = None, +) -> torch.Tensor: + """Absorbed MLA attention with a fused Triton score/softmax/latent core. + + ``softmax_scale`` defaults to ``1 / sqrt(Dn + Dr)`` where ``Dn`` is the + ``q_nope`` head dim and ``Dr`` the RoPE head dim, matching the materialized + MLA path. The fused core does not implement dense masks; a non-``None`` + ``attn_mask`` is rejected rather than silently ignored. + """ + if attn_mask is not None: + raise NotImplementedError( + "fused absorbed MLA does not support attn_mask; use " + "absorbed_mla_attention as the dense-mask reference" + ) + if q_nope.ndim != 4 or q_rope.ndim != 4: + raise ValueError("q_nope and q_rope must be [B, S, H, D]") + batch, query_len, num_heads, qk_nope_head_dim = q_nope.shape + qk_rope_head_dim = q_rope.size(-1) + if compressed_kv.ndim != 3 or compressed_kv.size(0) != batch: + raise ValueError("compressed_kv must be [B, T, K]") + if k_rope.shape != (batch, compressed_kv.size(1), qk_rope_head_dim): + raise ValueError("k_rope must be [B, T, qk_rope_head_dim]") + kv_lora_rank = compressed_kv.size(-1) + inferred_v_head_dim = kv_b_proj_weight.shape[0] // num_heads - qk_nope_head_dim + if inferred_v_head_dim <= 0 or kv_b_proj_weight.shape[0] % num_heads: + raise ValueError("kv_b_proj.weight rows do not describe per-head K/V") + w_uk, w_uv = split_kv_b_proj( + kv_b_proj_weight, + num_heads=num_heads, + qk_nope_head_dim=qk_nope_head_dim, + v_head_dim=inferred_v_head_dim, + kv_lora_rank=kv_lora_rank, + ) + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim) + + q_lat = torch.einsum("bshn,hnk->bshk", q_nope, w_uk) + out_latent = _AbsorbedMLAFusedCore.apply( + q_lat, q_rope, compressed_kv, k_rope, float(softmax_scale) + ) + output = torch.einsum("bshk,hvk->bshv", out_latent, w_uv) + return output.reshape(batch, query_len, num_heads * inferred_v_head_dim) diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index ca61cad..cd27b4f 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -15,7 +15,7 @@ frameworks require separate native environments and their disclosed revisions. | `tools.performance.deepspec_offline_benchmark` | Native cache preparation and disk-fed training | | `tools.performance.bench_eagle3_train_core` | EAGLE3 train-core latency and memory | | `tools.performance.bench_thd_cp_attention` | Packed attention with context parallelism | -| `tools.performance.bench_absorbed_mla` | Absorbed versus materialized MLA attention feasibility (one layer) | +| `tools.performance.bench_absorbed_mla` | Absorbed versus materialized MLA attention, including the fused Triton core (one layer) | Run `python -m --help` for arguments. Templates are listed in the [configuration index](../../configs/README.md). Supply your own model, initial @@ -77,6 +77,7 @@ completion are included. - [EAGLE3](eagle3-h200.md): synthetic train-core measurement and numerical scope. - [Packed attention/CP](cudnn-thd-cp.md): attention component results. - [Absorbed MLA](absorbed-mla.md): absorbed versus materialized attention identity and a negative training-shape result. +- [Absorbed MLA fused kernel](absorbed-mla-fused.md): fused Triton forward/backward for the absorbed identity, with memory and speed measurements. - [Training and acceptance](dspark-alignment.md): TorchSpec/ASpec BSHD/THD training alignment and verifier metrics. - [CP quality alignment](dspark-cp-alignment.md): CP2 versus same-layout CP1 diff --git a/docs/benchmarks/absorbed-mla-fused.md b/docs/benchmarks/absorbed-mla-fused.md new file mode 100644 index 0000000..4f1dfe8 --- /dev/null +++ b/docs/benchmarks/absorbed-mla-fused.md @@ -0,0 +1,95 @@ +# Absorbed MLA fused kernel (follow-up to #46) + +This note is the fused-kernel follow-up to +[absorbed-mla.md](absorbed-mla.md) and [ASpec #19](https://github.com/inclusionAI/ASpec/issues/19). +The dense reference in `aspec.models.ops.absorbed_mla` keeps `[B, H, S, T]` logits +materialized; the fused core keeps the same absorbed identity but never +materializes them: + +```text +q_lat = q_nope @ W_UK # [B, S, H, K] +logits = q_lat @ c_kv^T + q_rope @ k_rope^T +out = (softmax(logits) @ c_kv) @ W_UV^T +``` + +`aspec.models.ops.absorbed_mla_fused` adds a FlashAttention-style Triton +forward (online softmax accumulating the latent output) and a deterministic +backward. It is a prototype: dense masks, packed/THD layouts, Direct-CP and +`o_proj` fusion are not covered, and the naive reference remains the numerical +oracle in `tests/cpu/test_absorbed_mla.py`. The wrapper rejects a non-`None` +`attn_mask` with `NotImplementedError` instead of silently ignoring it. + +## Kernel design + +- Forward: one program per `[BLOCK_M, H]` query tile walks all keys, computes + `q_lat @ c_kv^T + q_rope @ k_rope^T`, and maintains `m`/`l`/`out_lat` in FP32. + Saved state is `out_lat` and `lse` only. +- Backward: two kernels, both recomputing logits from `lse`. + - query-side: `dq_lat`/`dq_rope` per query tile, no atomics; + - key/value-side: `dc_kv`/`dk_rope` per key tile and head group. `c_kv` is + shared across heads, so each program owns `H / GROUP` heads and writes a + per-group FP32 partial; a deterministic reduction sums the groups. This + avoids both the serial all-heads loop and atomics. +- The input projections (`q_nope @ W_UK`, `out_lat @ W_UV`) stay in PyTorch so + their gradients use normal autograd. +- BF16/FP16, contiguous BSHD/BTD inputs. The configured query tile is capped at + the draft block length to avoid wasted work on short blocks. + +Tiles were selected on the measurement host: forward `32x64`, query-side +`16x64`, key/value-side `64x32`, 4 warps; head group 8. + +## Measurement + +`tools.performance.bench_absorbed_mla` now includes `absorbed_fused` next to +`materialized_sdpa`, `materialized_explicit` and `absorbed`. It measures one +layer forward+backward at the attention boundary: BF16, `B=1`, `H=32`, +`Dn=128`, `Dr=64`, `Dv=128`, `K=512`, block 7, no mask, `o_proj`, Target +capture or optimizer step. Peak memory is `torch.cuda.max_memory_allocated()`. + +H100 PCIe 80GB, driver 580.126.09, PyTorch 2.13.0+cu130, Triton 3.7.1, warmup 2, +5 runs, median: + +| context | anchors | `S` | `T` | materialized SDPA | absorbed (PyTorch) | absorbed fused | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 4096 | 8 | 56 | 4152 | 3.37 ms / 0.245 GiB | 4.48 ms / 0.225 GiB | 3.86 ms / 0.137 GiB | +| 4096 | 128 | 896 | 4992 | 2.60 ms / 0.348 GiB | 10.30 ms / 2.799 GiB | 9.18 ms / 0.370 GiB | +| 4096 | 512 | 3584 | 7680 | 9.20 ms / 0.668 GiB | 63.64 ms / 16.688 GiB | 52.27 ms / 1.245 GiB | +| 16384 | 128 | 896 | 17280 | 6.39 ms / 0.830 GiB | 34.70 ms / 9.387 GiB | 28.59 ms / 0.449 GiB | +| 16384 | 256 | 1792 | 18176 | 11.04 ms / 0.937 GiB | 73.12 ms / 19.621 GiB | 59.30 ms / 0.675 GiB | + +## Result + +- The fused kernel is consistently faster than the naive PyTorch absorbed form + at every measured anchor budget (about 1.1–1.25x), and the gap grows with + `S` because the naive form materializes `[B, H, S, T]` logits. +- Peak memory collapses: 13.4x less than the naive absorbed form at the default + 512-anchor shape (1.245 vs 16.688 GiB) and 29x less at 16k context with 256 + anchors (0.675 vs 19.621 GiB). That is the main result: absorbed execution + trades extra FLOPs for activation memory, and a fused kernel makes the trade + usable. +- It is still slower than materialized SDPA (1.15–5.7x). The absorbed form does + about 3.4x more attention math at this geometry (`K+Dr` scores and `K` latent + output versus `192`/`128` materialized), and backward recomputes logits in two + passes. A kernel cannot remove that algorithmic gap on the default workload; + the memory reduction is the reason to consider it. + +Reproduce: + +```bash +env -u LD_LIBRARY_PATH CUDA_VISIBLE_DEVICES=1 \ + python -m tools.performance.bench_absorbed_mla \ + --context-length 4096 --num-anchors 512 --block-size 7 \ + --warmup 2 --runs 5 +``` + +Keep raw JSON outside the source tree. `LD_LIBRARY_PATH` was unset only to +select this PyTorch build's bundled cuDNN on the measurement host. + +## Remaining work + +The first gap versus the dense reference is boolean `attn_mask` support, +including fully masked rows; the fused wrapper currently fails closed on it. +Beyond that: FlexAttention/cuDNN masks, packed THD and Direct-CP layouts, +`o_proj` fusion, shape-based tile dispatch/autotuning, end-to-end memory/time +with Target capture, exact resume and export. None of these are established by +this note. diff --git a/tests/test_absorbed_mla_fused_gpu.py b/tests/test_absorbed_mla_fused_gpu.py new file mode 100644 index 0000000..9061ef4 --- /dev/null +++ b/tests/test_absorbed_mla_fused_gpu.py @@ -0,0 +1,148 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU checks for the fused Triton absorbed-MLA attention core. + +These tests compare the fused kernel against the dense PyTorch reference in +``aspec.models.ops.absorbed_mla``. They require a CUDA device and Triton; the +CPU suite skips them. +""" + +import math + +import pytest +import torch + +from aspec.models.ops import absorbed_mla_fused as fused_ops +from aspec.models.ops.absorbed_mla import absorbed_mla_attention + +pytestmark = pytest.mark.skipif( + not fused_ops._HAS_TRITON or not torch.cuda.is_available(), + reason="fused absorbed MLA requires Triton and CUDA", +) + + +def _make_tensors(*, batch=1, query=8, key=41, heads=4, nope=16, rope=16, value=16, latent=32): + generator = torch.Generator().manual_seed(41) + return { + "q_nope": torch.randn( + batch, query, heads, nope, generator=generator + ).cuda().half(), + "q_rope": torch.randn( + batch, query, heads, rope, generator=generator + ).cuda().half(), + "compressed_kv": torch.randn( + batch, key, latent, generator=generator + ).cuda().half(), + "k_rope": torch.randn(batch, key, rope, generator=generator).cuda().half(), + "kv_b_proj_weight": torch.randn( + heads * (nope + value), latent, generator=generator + ).cuda().half(), + } + + +def _relative_error(actual, expected): + expected = expected.float() + return ( + (actual.float() - expected).norm() / expected.norm().clamp_min(1e-6) + ).item() + + +def _run_grads(fn, tensors): + values = {name: value.detach().clone().requires_grad_(True) for name, value in tensors.items()} + output = fn(values) + grads = torch.autograd.grad(output.float().pow(2).sum(), list(values.values())) + return output, grads + + +@pytest.mark.parametrize( + "shape", + [ + dict(batch=1, query=8, key=41, heads=2), + dict(batch=1, query=7, key=41, heads=3), + dict(batch=2, query=33, key=100, heads=8, latent=64), + dict(batch=1, query=70, key=200, heads=4, latent=128), + ], +) +def test_fused_forward_and_gradients_match_reference(shape): + tensors = _make_tensors(**shape) + reference = absorbed_mla_attention(**tensors) + fused = fused_ops.fused_absorbed_mla_attention(**tensors) + assert torch.isfinite(fused).all() + assert _relative_error(fused, reference) < 2e-2 + + _, reference_grads = _run_grads( + lambda values: absorbed_mla_attention(**values), tensors + ) + _, fused_grads = _run_grads( + lambda values: fused_ops.fused_absorbed_mla_attention(**values), tensors + ) + for name, expected, actual in zip( + tensors, reference_grads, fused_grads, strict=True + ): + assert torch.isfinite(actual).all(), name + assert _relative_error(actual, expected) < 3e-2, name + + +def test_fused_default_scale_matches_materialized(): + tensors = _make_tensors() + fused = fused_ops.fused_absorbed_mla_attention(**tensors) + explicit = fused_ops.fused_absorbed_mla_attention( + **tensors, + softmax_scale=1.0 / math.sqrt(16 + 16), + ) + torch.testing.assert_close(fused, explicit, rtol=0, atol=0) + + +def test_fused_core_matches_float32_reference_at_latent_512(): + torch.manual_seed(7) + query, key, heads, latent, rope = 8, 64, 2, 512, 64 + q_lat = torch.randn(1, query, heads, latent, device="cuda", dtype=torch.bfloat16) + q_rope = torch.randn(1, query, heads, rope, device="cuda", dtype=torch.bfloat16) + c_kv = torch.randn(1, key, latent, device="cuda", dtype=torch.bfloat16) + k_rope = torch.randn(1, key, rope, device="cuda", dtype=torch.bfloat16) + scale = 1.0 / math.sqrt(128 + 64) + + def reference(q_lat, q_rope, c_kv, k_rope): + scores = torch.einsum("bshk,btk->bhst", q_lat.float(), c_kv.float()) + scores = scores + torch.einsum("bshr,btr->bhst", q_rope.float(), k_rope.float()) + probs = torch.softmax(scores * scale, dim=-1) + return torch.einsum("bhst,btk->bshk", probs, c_kv.float()) + + values = [q_lat, q_rope, c_kv, k_rope] + expected = reference(*values) + actual = fused_ops.fused_absorbed_mla_core( + q_lat, q_rope, c_kv, k_rope, softmax_scale=scale + ) + assert torch.isfinite(actual).all() + assert _relative_error(actual, expected) < 1e-2 + + fused_values = [value.detach().clone().requires_grad_(True) for value in values] + ref_values = [value.detach().clone().requires_grad_(True) for value in values] + ref_out = reference(*ref_values) + fused_out = fused_ops.fused_absorbed_mla_core( + *fused_values, softmax_scale=scale + ) + ref_grads = torch.autograd.grad(ref_out.pow(2).sum(), ref_values) + fused_grads = torch.autograd.grad(fused_out.float().pow(2).sum(), fused_values) + for expected_grad, actual_grad in zip(ref_grads, fused_grads, strict=True): + assert torch.isfinite(actual_grad).all() + assert _relative_error(actual_grad, expected_grad) < 3e-2 + + +def test_fused_core_rejects_cpu_tensors(): + if not fused_ops._HAS_TRITON: + pytest.skip("triton is not installed") + tensor = torch.zeros(1, 2, 1, 16) + with pytest.raises(RuntimeError, match="CUDA"): + fused_ops.fused_absorbed_mla_core( + tensor, tensor, torch.zeros(1, 2, 16), torch.zeros(1, 2, 16), + softmax_scale=0.1, + ) + + +def test_fused_rejects_dense_mask(): + tensors = _make_tensors() + mask = torch.ones(1, 8, 4, 41, dtype=torch.bool, device="cuda") + with pytest.raises(NotImplementedError, match="attn_mask"): + fused_ops.fused_absorbed_mla_attention(**tensors, attn_mask=mask) diff --git a/tools/performance/bench_absorbed_mla.py b/tools/performance/bench_absorbed_mla.py index 2f6de1b..b3eb101 100644 --- a/tools/performance/bench_absorbed_mla.py +++ b/tools/performance/bench_absorbed_mla.py @@ -20,7 +20,9 @@ - ``materialized_sdpa``: expand ``kv_b_proj`` and call fused SDPA; - ``materialized_explicit``: expand ``kv_b_proj`` and materialize logits; -- ``absorbed``: keep the normalized latent and re-associate both projections. +- ``absorbed``: keep the normalized latent and re-associate both projections; +- ``absorbed_fused``: the same absorbed identity with the fused Triton + score/softmax/latent core (when Triton and CUDA are available). FlexAttention/cuDNN masks, packed layouts and Direct-CP are outside this benchmark. Report the boundary together with the result; this does not establish @@ -41,6 +43,10 @@ import torch.nn.functional as F from aspec.models.ops.absorbed_mla import absorbed_mla_attention +from aspec.models.ops.absorbed_mla_fused import ( + _HAS_TRITON, + fused_absorbed_mla_attention, +) @dataclass(frozen=True) @@ -164,6 +170,14 @@ def absorbed(tensors, workload: Workload): ) +def absorbed_fused(tensors, workload: Workload): + return fused_absorbed_mla_attention( + tensors["q_nope"], tensors["q_rope"], tensors["compressed_kv"], + tensors["k_rope"], tensors["kv_b_proj_weight"], + softmax_scale=workload.softmax_scale, + ) + + def run_step(fn, tensors, workload): output = fn(tensors, workload) loss = output.float().pow(2).sum() @@ -238,6 +252,8 @@ def main(argv=None) -> int: "materialized_explicit": materialized_explicit, "absorbed": absorbed, } + if _HAS_TRITON: + variants["absorbed_fused"] = absorbed_fused results = { "workload": asdict(workload) | { "query_length": workload.query_length,