From 4039170a3e98c5304971a8e02a292ccdeb125780 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:53:08 +0800 Subject: [PATCH 1/3] feat(jetspec): add causal parallel Draft model and forward-KL loss Add a JetSpec (arXiv:2606.18394) model/loss core on the retained DFlash block head: causal intra-block attention plus temperature-scaled forward KL against frozen Target features. Reuse DFlash anchor sampling, masks and reductions through a new per-token loss hook. No trainer selection, tree builder, export or serving consumer is added. Part of #15. --- aspec/models/draft/jetspec.py | 42 +++++ aspec/models/jetspec.py | 98 +++++++++++ docs/framework/REFERENCES.md | 1 + docs/support.md | 1 + docs/user-guide/algorithms.md | 8 + docs/user-guide/jetspec.md | 111 ++++++++++++ tests/cpu/test_jetspec.py | 317 ++++++++++++++++++++++++++++++++++ 7 files changed, 578 insertions(+) create mode 100644 aspec/models/draft/jetspec.py create mode 100644 aspec/models/jetspec.py create mode 100644 docs/user-guide/jetspec.md create mode 100644 tests/cpu/test_jetspec.py diff --git a/aspec/models/draft/jetspec.py b/aspec/models/draft/jetspec.py new file mode 100644 index 0000000..ce6d0b2 --- /dev/null +++ b/aspec/models/draft/jetspec.py @@ -0,0 +1,42 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# The JetSpec method is described in "JetSpec: Breaking the Scaling Ceiling of +# Speculative Decoding with Parallel Tree Drafting" (Hu et al., 2026, +# arXiv:2606.18394). This is an independent ASpec implementation: the retained +# DFlash block head already provides the selected-Target-layer feature fusion and +# dual-source KV injection; JetSpec fixes causal intra-block attention. No +# JetSpec source file is copied. +"""JetSpec causal-parallel Draft head over frozen Target features.""" + +from aspec.models.draft.dflash import DFlashConfig, DFlashDraftModel + + +class JetSpecConfig(DFlashConfig): + """Causal-parallel Draft configuration with an explicit block contract.""" + + model_type = "jetspec" + + def __init__(self, *, is_causal=True, block_size=16, **kwargs): + if is_causal is not True: + raise ValueError("JetSpec requires causal intra-block attention") + if isinstance(block_size, bool) or not isinstance(block_size, int) or block_size < 2: + raise ValueError("JetSpec block_size must be an integer >= 2") + architectures = kwargs.pop("architectures", ["JetSpecDraftModel"]) + if architectures != ["JetSpecDraftModel"]: + raise ValueError("JetSpec uses exactly one JetSpecDraftModel architecture") + kwargs["is_causal"] = True + super().__init__(architectures=architectures, block_size=block_size, **kwargs) + if self.ffn_type != "dense": + raise ValueError("JetSpec currently supports dense Draft FFNs only") + layer_types = list(getattr(self, "layer_types", []) or []) + if any(kind != "full_attention" for kind in layer_types): + raise ValueError("JetSpec currently supports full intra-block attention only") + if getattr(self, "mtp_conditioning", None) is not None: + raise ValueError("JetSpec does not use the native-MTP conditioning contract") + + +class JetSpecDraftModel(DFlashDraftModel): + """DFlash block head with the JetSpec causal configuration.""" + + config_class = JetSpecConfig diff --git a/aspec/models/jetspec.py b/aspec/models/jetspec.py new file mode 100644 index 0000000..91fed34 --- /dev/null +++ b/aspec/models/jetspec.py @@ -0,0 +1,98 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Objective reimplemented from the public JetSpec paper (Hu et al., 2026, +# arXiv:2606.18394, Section 3.3 and Appendix D): temperature-scaled forward KL +# between the frozen Target and the causal parallel Draft, with optional +# DFlash-style depth weighting and the T_KD^2 gradient scale. Configuration, +# anchor sampling, masks, reductions and recovery stay in the ASpec DFlash path. +"""JetSpec causal-parallel Draft training objective.""" + +import math + +import torch +import torch.nn.functional as F + +from aspec.models.dflash import DFlashModel + + +class JetSpecModel(DFlashModel): + """DFlash block head with causal intra-block attention and forward KL.""" + + def __init__(self, *args, distillation_temperature=1.0, **kwargs): + # Main JetSpec results use uniform per-position weights (gamma = 0). + kwargs.setdefault("loss_decay_gamma", 0.0) + super().__init__(*args, **kwargs) + config = getattr(self.draft_model, "config", None) + if not bool(getattr(config, "is_causal", False)): + raise ValueError("JetSpec requires Draft config is_causal=true") + if self.mtp_conditioning is not None: + raise ValueError("JetSpec does not use the native-MTP conditioning contract") + if self.loss_objective != "decay": + raise ValueError("JetSpec supports the decay/uniform weighting objective only") + if self.ce_loss_alpha != 0 or self.l1_loss_alpha != 0: + raise ValueError("JetSpec replaces CE/L1 with forward KL; set both alphas to zero") + if self.prediction_start != 1: + raise ValueError("JetSpec supervises the suffix after the anchor position") + if (isinstance(distillation_temperature, bool) + or not isinstance(distillation_temperature, (int, float)) + or not math.isfinite(float(distillation_temperature)) + or float(distillation_temperature) <= 0): + raise ValueError("JetSpec distillation_temperature must be positive and finite") + self.distillation_temperature = float(distillation_temperature) + if config is not None and hasattr(config, "block_size"): + if int(config.block_size) != self.block_size: + raise ValueError("JetSpec training block_size must match its Draft config") + + @property + def uses_target_distribution(self) -> bool: + return True + + @property + def uses_target_hidden_states(self) -> bool: + return True + + def _block_mask_options(self) -> dict: + return {"is_causal": True} + + def _block_mask_options_for_layer(self, layer_id: int) -> dict: + return {"is_causal": True} + + def _training_loss( + self, + *, + logits, + draft_hidden, + target_ids, + last_hidden_states, + lm_head_weight, + safe_label_indices, + packed_context_positions, + sequence_length, + input_ids, + anchor_positions, + ): + if last_hidden_states is None: + raise ValueError( + "JetSpec forward KL requires Target last_hidden_states; set " + "inference.store_last_hidden_states=true in the run config" + ) + target_positions = (safe_label_indices - 1).clamp(min=0) + aligned_hidden = self._align_target_hidden_states( + last_hidden_states, + target_positions, + global_sequence_length=sequence_length, + packed_context_positions=packed_context_positions, + ) + with torch.no_grad(): + target_logits = F.linear( + aligned_hidden.detach(), lm_head_weight.detach() + ).float() + draft_logits = logits.float() + temperature = self.distillation_temperature + log_q = F.log_softmax(draft_logits / temperature, dim=-1) + p = F.softmax(target_logits / temperature, dim=-1) + kl = F.kl_div(log_q, p, reduction="none").sum(-1) + # The paper's Eq. 6 keeps the standard T^2 gradient scale. + kl = kl * (temperature * temperature) + return kl, kl, logits diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index d61105b..fa68237 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -20,6 +20,7 @@ the retained source identities; applicable license and notice texts are distribu | [Transformers](https://github.com/huggingface/transformers) v4.57.1 `8cb5963cc22174954e7dca2c0a3320b7dc2f4edc` | Adapted `Qwen3MoeSparseMoeBlock` and load-balancing objective in `aspec/models/draft/moe.py` | | [DMTP](https://sylvan820.github.io/D-MTP/) | Inspiration for dual-anchor MTP hidden conditioning and zero-initialized embedding residual; independent ASpec integration reuses the retained native predictor, without copying DMTP code | | [SpecForge DMTP capture](https://github.com/sgl-project/SpecForge/tree/c02184448f9b000060df50307b7044c3b475ad79) | Inspiration for Target-side frozen MTP feature production; ASpec adapts its own native predictor and publication/transport lifecycle, without copying the capture plugin | +| [JetSpec](https://github.com/hao-ai-lab/JetSpec) and [arXiv:2606.18394](https://arxiv.org/abs/2606.18394) | Public method reference for causal parallel Draft attention and temperature-scaled forward KL; independently implemented on the retained DFlash block head, without copying JetSpec source | | [Speculators](https://github.com/vllm-project/speculators) `7ec7aba9450b533334e8f7992baf4e93be3c1a57` | Native MTP recurrent objective and checkpoint bindings; ASpec supplies its own training/layout integration | | [Speculators](https://github.com/vllm-project/speculators) `3702320434bbdaa853f0846a66e6b6b8f1d97aec` | Adapted P-EAGLE model/loss and Qwen3 first-layer fusion; deterministic dense layout and ASpec input/observation boundary | | [Domino](https://github.com/jianuo-huang/Domino) `44e1e6912ba9d301899739a54bfda2ce8826980a` and [arXiv:2605.29707](https://arxiv.org/abs/2605.29707) | Public method and label-layout reference for the GRU correction head; independently implemented on the retained DFlash block backbone | diff --git a/docs/support.md b/docs/support.md index 1c01a98..7a5a153 100644 --- a/docs/support.md +++ b/docs/support.md @@ -15,6 +15,7 @@ and tests do not imply CPU training support. | Domino core | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts, eager/SDPA | Model/loss prerequisite only; dense full attention, CP1 padded execution; no trainer, curriculum schedule, resume, export or consumer validation; see [scope](user-guide/domino.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | +| JetSpec core | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery | Model/loss prerequisite only; no trainer selection, tree builder, recovery, export or consumer validation; see [scope](user-guide/jetspec.md) | | MTP-conditioned DSpark / DFlash | Target-side frozen native Qwen3.5 features, token/hidden dual-anchor training, exact recovery and strict native assets; legacy Draft-side assets retained | Dense full GQA, CP1, MBS1, REPLICATE, one node, non-packing, decay objective; Target capture PP1/unquantized; no generic serving consumer | | Native MTP | Qwen3.5-9B native single predictor, same-Target frozen embedding/head, multi-step training, recovery and export/stitch | Text-only Dense MTP; SDPA CP1 or independently validated FA4 packing/CP; no generic multi-layer/MoE MTP | | Dense / Qwen MoE / Ling MoE | Structure-specific FFNs, routing state, checkpointing and strict assets | No promise of every algorithm × structure combination | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index b895b63..9c91b58 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -112,6 +112,14 @@ anchor-shared window. FA4 preserves convolution, selector, document boundaries and CP ownership. Functional results do not establish official-recipe reproduction or long-term quality. +## JetSpec core prerequisite + +The [JetSpec model/loss API](jetspec.md) combines the retained DFlash block head +with causal intra-block attention and a temperature-scaled forward-KL objective +against frozen Target features. `JetSpecConfig` forces that causal contract and +rejects incompatible structures. Public training, tree construction, recovery, +export and consumer validation remain pending. + ## Native MTP conditioning for DSpark/DFlash `configs/recipes/qwen3_5_9b_dmtp.yaml` uses the native Qwen3.5 predictor as a frozen diff --git a/docs/user-guide/jetspec.md b/docs/user-guide/jetspec.md new file mode 100644 index 0000000..d754d19 --- /dev/null +++ b/docs/user-guide/jetspec.md @@ -0,0 +1,111 @@ +# JetSpec model and loss core + +This is the first prerequisite for [ASpec #15](https://github.com/inclusionAI/ASpec/issues/15). +It provides `aspec.models.jetspec.JetSpecModel`, +`aspec.models.draft.jetspec.JetSpecConfig` and +`aspec.models.draft.jetspec.JetSpecDraftModel`. It is a model/loss API; the ASpec +training CLI does not select this algorithm yet. + +## Reference and selected method + +The implementation targets **JetSpec**, not JetSpec++. The selected reference is +[JetSpec: Breaking the Scaling Ceiling of Speculative Decoding with Parallel Tree +Drafting](https://arxiv.org/abs/2606.18394) (Hu et al., 2026), Section 3.3 and +Appendix D. The upstream [implementation](https://github.com/hao-ai-lab/JetSpec) is +MIT-licensed; this is an independent ASpec implementation of the published +objective, and no JetSpec source file is copied. + +JetSpec's Draft head injects fused features from selected frozen Target layers as +contextual K/V state and predicts a whole block in one pass. ASpec already retains +that head as the DFlash block head, including dual-source KV and anchor sampling. +The JetSpec core therefore adapts the retained DFlash path instead of duplicating +it, and adds the two pieces that distinguish JetSpec: + +- **causal intra-block attention** (`is_causal=true`), so each Draft position sees + the verified prefix, its anchor and earlier positions in the same block, but no + future Draft position and no other block; and +- a **temperature-scaled forward-KL distillation objective** against the frozen + Target, rather than CE or L1. + +The DFlash2 selector, MoE Draft FFNs, sliding-window Draft layers, native-MTP +conditioning and packed/CP execution are outside this core. + +## Prediction and supervision contract + +A block is anchored at an original sequence position `a`. Position `0` keeps the +real token `x[a]` as block context and is excluded from the loss. Position `k >= 1` +predicts `x[a+k]` and distills the Target distribution obtained from Target hidden +state `h[a+k-1]` and the frozen Target LM head. This is the next-token alignment +used by the retained DFlash path. + +The loss is + +```text +p = softmax(target_logits / T) +q = softmax(draft_logits / T) +L = T^2 * sum(w * KL(p || q)) / sum(w) +``` + +where `w` includes block validity, sequence bounds, `loss_mask`, the anchor +exclusion, and optional depth weighting `exp(-(k-1)/gamma)`. The default +`distillation_temperature=1.0` and `loss_decay_gamma=0.0` (uniform weights) match +the main paper configuration; both are configurable. Target logits are produced +under `torch.no_grad()`. The Target hidden states must come from +`inference.store_last_hidden_states=true`, as for the existing L1 distillation path. + +`JetSpecModel` fails closed when the Draft config is not causal, when CE/L1 alphas +are nonzero, when the training block size disagrees with the Draft config, or when +the temperature is not positive and finite. `JetSpecConfig` fixes the architecture +to `JetSpecDraftModel`, forces `is_causal=true`, and rejects non-dense FFNs, +sliding-window layers and MTP conditioning. + +## Minimal numerical use + +Inside the [isolated environment](installation.md): + +```python +import torch +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel +from aspec.models.jetspec import JetSpecModel + +config = JetSpecConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, +) +draft = JetSpecDraftModel(config) +draft.embed_tokens.requires_grad_(False) +model = JetSpecModel( + draft, block_size=5, num_anchors=2, attention_backend="sdpa", + ce_loss_alpha=0.0, l1_loss_alpha=0.0, distillation_temperature=1.0, +) + +ids = torch.arange(12)[None] +outputs = model( + input_ids=ids, + hidden_states_list=torch.randn(1, 12, 16), # selected Target layers + loss_mask=torch.ones_like(ids), + lm_head_weight=torch.randn(32, 16), # frozen Target head + last_hidden_states=torch.randn(1, 12, 16), # frozen Target final state + injected_anchors=torch.tensor([[3, 5]]), + injected_keep_mask=torch.ones(1, 2, dtype=torch.bool), +) +outputs[0].backward() +``` + +The return value follows `DFlashModel.forward`: scalar loss, accuracy, +per-position loss/accuracy/count, extra components and the objective numerator / +denominator. Per-position loss now reports KL rather than CE. + +## Boundaries and validation + +- This core does not provide a recipe, trainer selection, exact resume, tree + construction, export, strict reloading or a serving consumer. Those remain for + follow-up work under the issue's expected outcomes. +- CPU checks cover the configuration contract, causal mask construction, forward + KL and gradients against independent equations, anchor/loss-mask exclusion, + empty supervision and document-local label alignment. They do not certify GPU + attention execution, convergence, acceptance length or serving performance. +- The default tests are in `tests/cpu/test_jetspec.py`. diff --git a/tests/cpu/test_jetspec.py b/tests/cpu/test_jetspec.py new file mode 100644 index 0000000..35bdeb3 --- /dev/null +++ b/tests/cpu/test_jetspec.py @@ -0,0 +1,317 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""JetSpec causal-parallel head and forward-KL objective. + +Reference: JetSpec, arXiv:2606.18394 (Section 3.3 and Appendix D). These checks +compare the implementation with explicit equations; they do not reproduce the +paper's training data, budget or serving results. +""" + +import copy + +import pytest +import torch +from torch.nn import functional as F + +from aspec.models.dflash import _create_dflash_mask_mod +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel +from aspec.models.jetspec import JetSpecModel + + +def make_config(**overrides): + values = dict( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, rms_norm_eps=1e-6, + ) + values.update(overrides) + return JetSpecConfig(**values) + + +def make_model(*, temperature=1.0, loss_decay_gamma=0.0, seed=17, **config_overrides): + with torch.random.fork_rng(devices=[]): + torch.random.default_generator.manual_seed(seed) + draft = JetSpecDraftModel(make_config(**config_overrides)) + draft.embed_tokens.requires_grad_(False) + return JetSpecModel( + draft, block_size=draft.config.block_size, num_anchors=2, + attention_backend="sdpa", ce_loss_alpha=0.0, l1_loss_alpha=0.0, + loss_decay_gamma=loss_decay_gamma, + distillation_temperature=temperature, + ) + + +def make_inputs(seed=31, length=12): + generator = torch.Generator().manual_seed(seed) + ids = torch.arange(length)[None] + return dict( + input_ids=ids, + hidden_states_list=torch.randn(1, length, 16, generator=generator), + loss_mask=torch.ones_like(ids), + lm_head_weight=torch.randn(32, 16, generator=generator), + last_hidden_states=torch.randn(1, length, 16, generator=generator), + injected_anchors=torch.tensor([[3, 5]]), + injected_keep_mask=torch.ones(1, 2, dtype=torch.bool), + ) + + +def run_and_capture(model, inputs): + captured = [] + handle = model.draft_model.register_forward_hook( + lambda _, __, output: captured.append(output) + ) + result = model(**inputs) + handle.remove() + return result, captured[0] + + +def reference_kl(model, draft_hidden, inputs): + """Dense forward-KL equations with explicit label/anchor/mask alignment.""" + anchors = inputs["injected_anchors"] + keep = inputs["injected_keep_mask"] + ids = inputs["input_ids"] + offsets = torch.arange(model.block_size)[None, None] + label_indices = anchors.unsqueeze(-1) + offsets + valid = label_indices < ids.shape[1] + safe_indices = label_indices.clamp(max=ids.shape[1] - 1) + + expanded_ids = ids.unsqueeze(1).expand(-1, anchors.shape[1], -1) + target_ids = torch.gather(expanded_ids, 2, safe_indices) + teacher_positions = (safe_indices - 1).clamp(min=0) + teacher_hidden = torch.gather( + inputs["last_hidden_states"].detach(), 1, + teacher_positions.reshape(1, -1, 1).expand(-1, -1, 16), + ) + target_logits = F.linear(teacher_hidden, inputs["lm_head_weight"].detach()).float() + draft_logits = F.linear(draft_hidden, inputs["lm_head_weight"]).float() + + temperature = model.distillation_temperature + log_q = F.log_softmax(draft_logits / temperature, dim=-1) + p = F.softmax(target_logits / temperature, dim=-1) + kl = F.kl_div(log_q, p, reduction="none").sum(-1) + kl = kl * (temperature * temperature) + + gathered_mask = torch.gather( + inputs["loss_mask"].unsqueeze(1).expand(-1, anchors.shape[1], -1), + 2, safe_indices, + ) + positions = offsets.squeeze(0) + weights = keep.unsqueeze(-1).float() * valid.float() + weights = weights * (positions >= model.prediction_start).float() + weights = weights * gathered_mask + if model.loss_decay_gamma is not None and model.loss_decay_gamma > 0: + weights = weights * torch.exp( + -(positions - model.prediction_start).clamp_min(0).float() + / model.loss_decay_gamma + ) + kl = kl.reshape_as(weights) + loss = (kl * weights).sum() / weights.sum() + assert target_ids.shape == kl.shape + return loss, kl + + +def _assert_parameter_gradients(actual, expected): + actual_parameters = dict(actual.named_parameters()) + expected_parameters = dict(expected.named_parameters()) + assert actual_parameters.keys() == expected_parameters.keys() + for name, parameter in actual_parameters.items(): + reference = expected_parameters[name] + assert (parameter.grad is None) == (reference.grad is None), name + if parameter.grad is not None: + assert torch.isfinite(parameter.grad).all(), name + torch.testing.assert_close( + parameter.grad, reference.grad, rtol=2e-5, atol=2e-6, msg=name + ) + + +def test_config_pins_the_causal_block_contract(): + config = make_config() + assert config.model_type == "jetspec" + assert config.architectures == ["JetSpecDraftModel"] + assert config.is_causal is True + assert config.block_size == 5 + assert config.layer_types == ["full_attention"] + restored = JetSpecConfig.from_dict(config.to_dict()) + assert restored.is_causal is True + assert restored.block_size == 5 + assert restored.architectures == ["JetSpecDraftModel"] + + +@pytest.mark.parametrize("override", [ + {"is_causal": False}, + {"is_causal": None}, + {"block_size": 1}, + {"block_size": 4.5}, + {"architectures": ["DFlashDraftModel"]}, +]) +def test_config_fail_closed(override): + with pytest.raises(ValueError): + make_config(**override) + + +def test_config_rejects_unsupported_head_shapes(): + with pytest.raises(ValueError): + make_config(ffn_type="moe", num_experts=4, num_experts_per_tok=2, + moe_intermediate_size=32) + with pytest.raises(ValueError): + make_config(layer_types=["sliding_attention"], sliding_window=64) + + +def test_blocks_use_causal_intra_block_attention(): + model = make_model() + assert model._block_mask_options_for_layer(0) == {"is_causal": True} + mask = _create_dflash_mask_mod( + torch.tensor([[3, 8]]), torch.ones(1, 2, dtype=torch.bool), + ctx_len=10, block_size=5, is_causal=True, + ) + # Query positions are block-relative; KV is [context | block 0 | block 1]. + for query in range(5): + assert mask(0, 0, query, 2) # verified prefix before the anchor + assert not mask(0, 0, query, 3) # anchor is draft context, not target context + assert mask(0, 0, query, 10) # block 0 anchor + assert mask(0, 0, query, 10 + query) # own causal prefix + assert not mask(0, 0, query, 10 + query + 1) if query < 4 else True + assert not mask(0, 0, query, 15) # another block is invisible + assert not mask(0, 0, 0, 11) # no causal future inside block 0 + + +@pytest.mark.parametrize("temperature", [1.0, 2.0]) +@pytest.mark.parametrize("loss_decay_gamma", [0.0, 3.0]) +def test_loss_and_gradients_match_independent_equations(temperature, loss_decay_gamma): + actual = make_model(temperature=temperature, loss_decay_gamma=loss_decay_gamma) + reference = copy.deepcopy(actual) + inputs, reference_inputs = make_inputs(), make_inputs() + + result, captured = run_and_capture(actual, inputs) + expected, reference_captured = run_and_capture(reference, reference_inputs) + expected_loss, expected_per_position = reference_kl( + reference, reference_captured, reference_inputs + ) + actual_loss, actual_per_position = reference_kl(actual, captured, inputs) + + torch.testing.assert_close(result[0], expected[0], rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_loss, result[0], rtol=2e-5, atol=2e-6) + torch.testing.assert_close(expected_loss, expected[0], rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_per_position, expected_per_position, + rtol=2e-5, atol=2e-6) + assert result[2][0] == 0 and result[4][0] == 0 # anchor slot is not supervised + + result[0].backward() + expected_loss.backward() + _assert_parameter_gradients(actual, reference) + + +def test_loss_mask_excludes_unsupervised_labels(): + model = make_model() + inputs = make_inputs() + inputs["loss_mask"][0, 8] = 0 + result, captured = run_and_capture(model, inputs) + expected, per_position = reference_kl(model, captured, inputs) + torch.testing.assert_close(result[0], expected, rtol=2e-5, atol=2e-6) + assert int(result[4][1:].sum()) == 7 + assert int(result[4][0]) == 0 + assert float(result[2][0].detach()) == 0.0 + assert per_position.shape == (1, 2, model.block_size) + + +def test_empty_supervision_is_neutral_and_finite(): + model = make_model() + inputs = make_inputs() + inputs["loss_mask"] = torch.zeros_like(inputs["loss_mask"]) + inputs["injected_keep_mask"] = torch.zeros_like(inputs["injected_keep_mask"]) + result, _ = run_and_capture(model, inputs) + assert float(result[0].detach()) == 0.0 + assert torch.isfinite(result[0]) + assert int(result[4].sum()) == 0 + result[0].backward() + for parameter in model.parameters(): + assert parameter.grad is None or torch.count_nonzero(parameter.grad) == 0 + + +def test_target_features_are_detached_and_required(): + model = make_model() + inputs = make_inputs() + inputs["last_hidden_states"].requires_grad_(True) + result, _ = run_and_capture(model, inputs) + result[0].backward() + assert inputs["last_hidden_states"].grad is None + assert model.draft_model.embed_tokens.weight.grad is None + + missing = make_inputs() + missing["last_hidden_states"] = None + with pytest.raises(ValueError, match="forward KL requires Target last_hidden_states"): + model(**missing) + + +@pytest.mark.parametrize("temperature", [0.0, -1.0, float("nan"), float("inf"), True]) +def test_invalid_temperature_fails_closed(temperature): + draft = JetSpecDraftModel(make_config()) + with pytest.raises(ValueError, match="temperature"): + JetSpecModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa", + ce_loss_alpha=0.0, l1_loss_alpha=0.0, + distillation_temperature=temperature) + + +def test_non_kl_components_fail_closed(): + draft = JetSpecDraftModel(make_config()) + with pytest.raises(ValueError, match="CE/L1"): + JetSpecModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa") + with pytest.raises(ValueError, match="CE/L1"): + JetSpecModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa", + ce_loss_alpha=0.0, l1_loss_alpha=1.0) + with pytest.raises(ValueError, match="block_size"): + JetSpecModel(draft, block_size=4, num_anchors=2, attention_backend="sdpa", + ce_loss_alpha=0.0, l1_loss_alpha=0.0) + + +def test_model_requires_a_causal_draft_config(): + from aspec.models.draft.dflash import DFlashConfig, DFlashDraftModel + + config = DFlashConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + is_causal=False, + ) + draft = DFlashDraftModel(config) + with pytest.raises(ValueError, match="is_causal"): + JetSpecModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa", + ce_loss_alpha=0.0, l1_loss_alpha=0.0) + + +def test_short_sequences_keep_loss_finite(): + model = make_model(block_size=5) + inputs = make_inputs(length=6) + inputs["injected_anchors"] = torch.tensor([[2, 4]]) + result, captured = run_and_capture(model, inputs) + expected, _ = reference_kl(model, captured, inputs) + torch.testing.assert_close(result[0], expected, rtol=2e-5, atol=2e-6) + assert torch.isfinite(result[0]) + result[0].backward() + for parameter in model.parameters(): + if parameter.grad is not None: + assert torch.isfinite(parameter.grad).all() + + +def test_changing_a_later_block_does_not_change_an_earlier_block_loss(): + model = make_model() + inputs = make_inputs() + inputs["injected_anchors"] = torch.tensor([[2, 7]]) + _, base_hidden = run_and_capture(model, inputs) + _, base_per_block = reference_kl(model, base_hidden, inputs) + changed = {key: value.clone() if torch.is_tensor(value) else value + for key, value in inputs.items()} + # Mutate labels and teacher states only after the first block's suffix. + changed["input_ids"][0, 8:] = (changed["input_ids"][0, 8:] + 1) % 32 + changed["last_hidden_states"][0, 8:] = torch.randn_like( + changed["last_hidden_states"][0, 8:] + ) + _, moved_hidden = run_and_capture(model, changed) + _, moved_per_block = reference_kl(model, moved_hidden, changed) + torch.testing.assert_close(base_per_block[0, 0], moved_per_block[0, 0], + rtol=0, atol=0) + assert not torch.equal(base_per_block[0, 1], moved_per_block[0, 1]) From 48f7dfe973e64382dc88f1841de0d3b7567c3672 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:20:23 +0800 Subject: [PATCH 2/3] fix(jetspec): flatten KL and enforce CPU causal isolation Address review findings: flatten draft and Target logits before the KL so batch>1 matches the one-dimensional token-weight layout; materialize the block-causal mask for the non-CUDA SDPA fallback while preserving unmasked behavior for non-causal DFlash configs; add a two-row regression and a CPU forward isolation test, and perturb the later anchor in the block test. --- aspec/models/dflash.py | 71 ++++++++++++++++++++++++++++++++++++ aspec/models/jetspec.py | 4 +-- docs/user-guide/jetspec.md | 5 +++ tests/cpu/test_jetspec.py | 73 +++++++++++++++++++++++++++++++++----- 4 files changed, 143 insertions(+), 10 deletions(-) diff --git a/aspec/models/dflash.py b/aspec/models/dflash.py index 0bc1ad7..c705f5f 100644 --- a/aspec/models/dflash.py +++ b/aspec/models/dflash.py @@ -131,6 +131,51 @@ def dflash_mask_mod(b, h, q_idx, kv_idx): return dflash_mask_mod +def _create_dflash_dense_mask( + anchor_positions: torch.Tensor, + block_keep_mask: torch.Tensor, + ctx_len: int, + block_size: int, + is_causal: bool, +) -> torch.Tensor: + """Materialize the block mask for backends without FlexAttention (CPU). + + Returns a boolean ``[B, Q, KV]`` mask where True means visible. Invalid + blocks see only their own draft query row so softmax stays finite; their + objective weights are already zero. + """ + num_anchors = anchor_positions.shape[1] + draft_len = num_anchors * block_size + kv_len = ctx_len + draft_len + device = anchor_positions.device + q_index = torch.arange(draft_len, device=device) + kv_index = torch.arange(kv_len, device=device) + q_block = q_index // block_size + anchor = anchor_positions.index_select(1, q_block) + + is_context = kv_index < ctx_len + mask_context = is_context.unsqueeze(0).unsqueeze(0) & ( + kv_index.unsqueeze(0).unsqueeze(0) < anchor.unsqueeze(-1) + ) + is_draft = kv_index >= ctx_len + kv_block = (kv_index - ctx_len) // block_size + mask_draft = is_draft.unsqueeze(0).unsqueeze(0) & ( + q_block.unsqueeze(0).unsqueeze(-1) == kv_block.unsqueeze(0).unsqueeze(0) + ) + if is_causal: + q_offset = q_index % block_size + kv_offset = (kv_index - ctx_len) % block_size + mask_draft = mask_draft & ( + kv_offset.unsqueeze(0).unsqueeze(0) <= q_offset.unsqueeze(0).unsqueeze(-1) + ) + valid = block_keep_mask.index_select(1, q_block) + mask = (mask_context | mask_draft) & valid.unsqueeze(-1) + + diagonal = torch.zeros(draft_len, kv_len, dtype=torch.bool, device=device) + diagonal[torch.arange(draft_len, device=device), ctx_len + q_index] = True + return mask | (~valid.unsqueeze(-1) & diagonal.unsqueeze(0)) + + class DFlashModel(nn.Module): """DFlash training wrapper. @@ -820,6 +865,31 @@ def _draft_backbone( else block_masks ) + dense_attention_masks = None + if block_mask is None and direct_cp_metadata is None and device.type != "cuda": + dense_cache = {} + dense_masks = [] + for layer_id in range(self.draft_model.num_layers): + options = self._block_mask_options_for_layer(layer_id) + causal = bool(options.get("is_causal", False)) + if causal and set(options) - {"is_causal"}: + raise ValueError( + "CPU causal Draft attention does not implement these mask options" + ) + if causal not in dense_cache: + dense_cache[causal] = ( + _create_dflash_dense_mask( + anchor_positions, block_keep_mask, seq_len, + self.block_size, is_causal=True, + ) + if causal else None + ) + dense_masks.append(dense_cache[causal]) + if True in dense_cache: + dense_attention_masks = ( + dense_masks[0] if len(dense_cache) == 1 else dense_masks + ) + if direct_cp_metadata is not None: from dataclasses import replace @@ -946,6 +1016,7 @@ def _draft_backbone( draft_position_ids=draft_position_ids, context_position_ids=context_position_ids, block_mask=block_mask, + draft_attention_mask=dense_attention_masks, noise_embedding=noise_embedding, direct_cp_metadata=direct_cp_metadata, csa_metadata=csa_metadata, diff --git a/aspec/models/jetspec.py b/aspec/models/jetspec.py index 91fed34..eb4b44a 100644 --- a/aspec/models/jetspec.py +++ b/aspec/models/jetspec.py @@ -87,8 +87,8 @@ def _training_loss( with torch.no_grad(): target_logits = F.linear( aligned_hidden.detach(), lm_head_weight.detach() - ).float() - draft_logits = logits.float() + ).float().reshape(-1, logits.size(-1)) + draft_logits = logits.float().reshape(-1, logits.size(-1)) temperature = self.distillation_temperature log_q = F.log_softmax(draft_logits / temperature, dim=-1) p = F.softmax(target_logits / temperature, dim=-1) diff --git a/docs/user-guide/jetspec.md b/docs/user-guide/jetspec.md index d754d19..b924a53 100644 --- a/docs/user-guide/jetspec.md +++ b/docs/user-guide/jetspec.md @@ -30,6 +30,11 @@ it, and adds the two pieces that distinguish JetSpec: The DFlash2 selector, MoE Draft FFNs, sliding-window Draft layers, native-MTP conditioning and packed/CP execution are outside this core. +FlexAttention is used on CUDA. On the CPU SDPA fallback, the same block-causal +visibility is materialized explicitly so the isolation contract is exercised by +the default tests. Draft configurations that are not causal keep the previous +unmasked fallback behavior. + ## Prediction and supervision contract A block is anchored at an original sequence position `a`. Position `0` keeps the diff --git a/tests/cpu/test_jetspec.py b/tests/cpu/test_jetspec.py index 35bdeb3..9a109b5 100644 --- a/tests/cpu/test_jetspec.py +++ b/tests/cpu/test_jetspec.py @@ -44,17 +44,17 @@ def make_model(*, temperature=1.0, loss_decay_gamma=0.0, seed=17, **config_overr ) -def make_inputs(seed=31, length=12): +def make_inputs(seed=31, length=12, batch=1): generator = torch.Generator().manual_seed(seed) - ids = torch.arange(length)[None] + ids = (torch.arange(batch * length).reshape(batch, length) % 32) return dict( input_ids=ids, - hidden_states_list=torch.randn(1, length, 16, generator=generator), + hidden_states_list=torch.randn(batch, length, 16, generator=generator), loss_mask=torch.ones_like(ids), lm_head_weight=torch.randn(32, 16, generator=generator), - last_hidden_states=torch.randn(1, length, 16, generator=generator), - injected_anchors=torch.tensor([[3, 5]]), - injected_keep_mask=torch.ones(1, 2, dtype=torch.bool), + last_hidden_states=torch.randn(batch, length, 16, generator=generator), + injected_anchors=torch.tensor([[3, 5]]).expand(batch, -1), + injected_keep_mask=torch.ones(batch, 2, dtype=torch.bool), ) @@ -83,7 +83,7 @@ def reference_kl(model, draft_hidden, inputs): teacher_positions = (safe_indices - 1).clamp(min=0) teacher_hidden = torch.gather( inputs["last_hidden_states"].detach(), 1, - teacher_positions.reshape(1, -1, 1).expand(-1, -1, 16), + teacher_positions.reshape(ids.shape[0], -1, 1).expand(-1, -1, 16), ) target_logits = F.linear(teacher_hidden, inputs["lm_head_weight"].detach()).float() draft_logits = F.linear(draft_hidden, inputs["lm_head_weight"]).float() @@ -204,6 +204,62 @@ def test_loss_and_gradients_match_independent_equations(temperature, loss_decay_ _assert_parameter_gradients(actual, reference) +def test_two_row_forward_kl_matches_reference_and_backward(): + actual = make_model(temperature=1.5) + reference = copy.deepcopy(actual) + inputs, reference_inputs = make_inputs(batch=2), make_inputs(batch=2) + result, _ = run_and_capture(actual, inputs) + expected, reference_captured = run_and_capture(reference, reference_inputs) + expected_loss, _ = reference_kl(reference, reference_captured, reference_inputs) + assert result[0].ndim == 0 + torch.testing.assert_close(result[0], expected[0], rtol=2e-5, atol=2e-6) + torch.testing.assert_close(expected_loss, expected[0], rtol=2e-5, atol=2e-6) + result[0].backward() + expected_loss.backward() + _assert_parameter_gradients(actual, reference) + + +def test_cpu_forward_enforces_block_isolation(): + model = make_model() + inputs = make_inputs() + inputs["injected_anchors"] = torch.tensor([[2, 7]]) + _, base_hidden = run_and_capture(model, inputs) + changed = {key: value.clone() if torch.is_tensor(value) else value + for key, value in inputs.items()} + # Block 0 (anchor 2) sees neither position 7 nor context positions 4..5. + changed["input_ids"][0, 7] = (changed["input_ids"][0, 7] + 1) % 32 + changed["hidden_states_list"][0, 4:6] += 5.0 + _, moved_hidden = run_and_capture(model, changed) + base_blocks = base_hidden.reshape(1, 2, 5, -1) + moved_blocks = moved_hidden.reshape(1, 2, 5, -1) + torch.testing.assert_close(base_blocks[:, 0], moved_blocks[:, 0], rtol=0, atol=0) + assert not torch.equal(base_blocks[:, 1], moved_blocks[:, 1]) + + +def test_non_causal_cpu_fallback_remains_bidirectional(): + from aspec.models.dflash import DFlashModel + from aspec.models.draft.dflash import DFlashConfig, DFlashDraftModel + + config = DFlashConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + ) + draft = DFlashDraftModel(config) + draft.embed_tokens.requires_grad_(False) + model = DFlashModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa") + inputs = make_inputs() + inputs["injected_anchors"] = torch.tensor([[2, 7]]) + _, base_hidden = run_and_capture(model, inputs) + changed = {key: value.clone() if torch.is_tensor(value) else value + for key, value in inputs.items()} + changed["hidden_states_list"][0, 10:] += 5.0 + _, moved_hidden = run_and_capture(model, changed) + assert not torch.equal(base_hidden.reshape(1, 2, 5, -1)[:, 0], + moved_hidden.reshape(1, 2, 5, -1)[:, 0]) + + def test_loss_mask_excludes_unsupervised_labels(): model = make_model() inputs = make_inputs() @@ -305,7 +361,8 @@ def test_changing_a_later_block_does_not_change_an_earlier_block_loss(): _, base_per_block = reference_kl(model, base_hidden, inputs) changed = {key: value.clone() if torch.is_tensor(value) else value for key, value in inputs.items()} - # Mutate labels and teacher states only after the first block's suffix. + # Change block 1's anchor token and later teacher states only. + changed["input_ids"][0, 7] = (changed["input_ids"][0, 7] + 1) % 32 changed["input_ids"][0, 8:] = (changed["input_ids"][0, 8:] + 1) % 32 changed["last_hidden_states"][0, 8:] = torch.randn_like( changed["last_hidden_states"][0, 8:] From e60ded7a1b501e0d646cd5bd7dcda7b0446ce2dd Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:02:38 +0800 Subject: [PATCH 3/3] fix(jetspec): plumb the dense fallback mask through the Draft backbone The training wrapper already passes draft_attention_mask into the Draft model, but the attention/decoder/model signatures and the SDPA fallback did not accept it, so every forward raised TypeError. Thread the mask through and use it as the SDPA attn_mask in the non-CUDA fallback. --- aspec/models/draft/dflash.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/aspec/models/draft/dflash.py b/aspec/models/draft/dflash.py index 5408a24..e3ed8e4 100644 --- a/aspec/models/draft/dflash.py +++ b/aspec/models/draft/dflash.py @@ -436,6 +436,7 @@ def forward( draft_position_ids: torch.Tensor, context_position_ids: torch.Tensor, block_mask=None, + draft_attention_mask=None, direct_cp_metadata=None, csa_metadata=None, csa_prepared=None, @@ -453,6 +454,8 @@ def forward( draft_position_ids: [B, draft_len] — position IDs for draft tokens context_position_ids: [B, ctx_len] — position IDs for context tokens block_mask: FlexAttention BlockMask for block-causal attention + draft_attention_mask: Boolean [B, draft_len, ctx_len+draft_len] mask + for the non-FlexAttention fallback (for example CPU tests). """ bsz, draft_len, _ = draft_hidden.shape ctx_len = context_hidden.shape[1] @@ -954,10 +957,18 @@ def observe_fixed_backward(gradient): enable_gqa=True, ) else: - # Fallback: bidirectional attention (no mask) — expand KV for SDPA + # Fallback: SDPA with an optional materialized block mask (CPU). k = _repeat_kv(k, self.num_kv_groups) v = _repeat_kv(v, self.num_kv_groups) - attn_output = F.scaled_dot_product_attention(q, k, v, is_causal=False, dropout_p=0.0) + if draft_attention_mask is None: + attn_output = F.scaled_dot_product_attention( + q, k, v, is_causal=False, dropout_p=0.0, + ) + else: + attn_output = F.scaled_dot_product_attention( + q, k, v, attn_mask=draft_attention_mask.unsqueeze(1), + is_causal=False, dropout_p=0.0, + ) if not use_cudnn_flex: attn_output = attn_output.transpose(1, 2).contiguous() @@ -1011,6 +1022,7 @@ def forward( draft_position_ids: torch.Tensor, context_position_ids: torch.Tensor, block_mask=None, + draft_attention_mask=None, direct_cp_metadata=None, csa_metadata=None, csa_prepared=None, @@ -1037,6 +1049,8 @@ def forward( } if hasattr(self.self_attn, "dsa_layer"): qsa_kwargs.update(dsa_metadata=dsa_metadata, dsa_layout=dsa_layout) + if draft_attention_mask is not None: + qsa_kwargs["draft_attention_mask"] = draft_attention_mask draft_hidden = self.self_attn( draft_hidden=draft_hidden, context_hidden=context_hidden, @@ -1300,6 +1314,7 @@ def forward( draft_position_ids: torch.Tensor, context_position_ids: torch.Tensor, block_mask=None, + draft_attention_mask=None, noise_embedding: Optional[torch.Tensor] = None, direct_cp_metadata=None, csa_metadata=None, @@ -1392,6 +1407,9 @@ def forward( layer_block_mask = block_mask if isinstance(block_mask, (list, tuple)): layer_block_mask = block_mask[layer_id] + layer_draft_mask = draft_attention_mask + if isinstance(draft_attention_mask, (list, tuple)): + layer_draft_mask = draft_attention_mask[layer_id] layer_metadata = ( direct_cp_metadata[layer_id] if isinstance(direct_cp_metadata, (list, tuple)) else direct_cp_metadata @@ -1424,6 +1442,7 @@ def forward( draft_position_ids=draft_position_ids, context_position_ids=context_position_ids, block_mask=layer_block_mask, + draft_attention_mask=layer_draft_mask, direct_cp_metadata=layer_metadata, csa_metadata=csa_metadata, csa_prepared=csa_prepared,