diff --git a/aspec/models/dflash.py b/aspec/models/dflash.py index d26495a..0bc1ad7 100644 --- a/aspec/models/dflash.py +++ b/aspec/models/dflash.py @@ -143,6 +143,9 @@ class DFlashModel(nn.Module): """ prediction_start = 1 + # First label offset relative to an anchor. Base DFlash predicts the token at + # the anchor itself (masked out); shifted-layout variants start at anchor + 1. + label_start = 0 def __init__( self, @@ -438,6 +441,53 @@ def _extra_training_loss( ) -> tuple[torch.Tensor, dict]: return logits.new_zeros(()), {} + def _training_loss( + self, + *, + logits: torch.Tensor, + draft_hidden: torch.Tensor, + target_ids: torch.Tensor, + last_hidden_states: Optional[torch.Tensor], + lm_head_weight: torch.Tensor, + safe_label_indices: torch.Tensor, + packed_context_positions: Optional[torch.Tensor], + sequence_length: int, + input_ids: torch.Tensor, + anchor_positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Per-token objective and the logits used for reporting. + + Base DFlash applies ``ce_loss_alpha * CE + l1_loss_alpha * L1``. + Algorithm wrappers may replace the objective (and the logits used for + accuracy) without duplicating anchor, mask or reduction bookkeeping. + """ + vocab_size = logits.size(-1) + flat_logits = logits.view(-1, vocab_size) + flat_targets = target_ids.view(-1) + ce_per_token = F.cross_entropy(flat_logits, flat_targets, reduction="none") + + loss_per_token = self.ce_loss_alpha * ce_per_token + if self.uses_target_distribution: + if last_hidden_states is None: + raise ValueError( + "DFlash L1 distillation (l1_loss_alpha > 0) requires target " + "last_hidden_states; set inference.store_last_hidden_states=true in the " + "run config." + ) + tgt_idx = (safe_label_indices - 1).clamp(min=0) + aligned_hidden = self._align_target_hidden_states( + last_hidden_states, + tgt_idx, + global_sequence_length=sequence_length, + packed_context_positions=packed_context_positions, + ) + target_logits = F.linear(aligned_hidden, lm_head_weight).view(-1, vocab_size) + target_probs = torch.softmax(target_logits.float(), dim=-1) + draft_probs = torch.softmax(flat_logits.float(), dim=-1) + l1_per_token = (draft_probs - target_probs).abs().sum(dim=-1) + loss_per_token = loss_per_token + self.l1_loss_alpha * l1_per_token + return loss_per_token, ce_per_token, logits + def _align_target_hidden_states( self, last_hidden_states: torch.Tensor, @@ -1094,8 +1144,10 @@ def forward( logits = self._compute_logits(draft_hidden, lm_head_weight) # 8. Compute labels and weight mask (SpecForge pattern) - # Labels: same-position prediction (position k predicts token at anchor+k) - label_offsets = torch.arange(0, self.block_size, device=device).view(1, 1, -1) + # Labels: same-position prediction (position k predicts token at anchor + label_start + k) + label_offsets = torch.arange( + self.label_start, self.label_start + self.block_size, device=device + ).view(1, 1, -1) label_indices = anchor_positions.unsqueeze(-1) + label_offsets # [B, n_blocks, block_size] valid_label_mask = label_indices < seq_len safe_label_indices = label_indices.clamp(max=seq_len - 1) @@ -1127,34 +1179,24 @@ def forward( # our objective weighting is an addition to the training signal, not the metric. binary_eval_mask = weight_mask.view(-1) - # 9. Per-token loss: ce_loss_alpha*CE + l1_loss_alpha*L1. - vocab_size = logits.size(-1) - flat_logits = logits.view(-1, vocab_size) + # 9. Per-token objective, algorithm-specific logits and reporting values. + loss_per_token, loss_per_token_by_position, logits = self._training_loss( + logits=logits, + draft_hidden=draft_hidden, + target_ids=target_ids, + last_hidden_states=last_hidden_states, + lm_head_weight=lm_head_weight, + safe_label_indices=safe_label_indices, + packed_context_positions=packed_context_positions, + sequence_length=seq_len, + input_ids=input_ids, + anchor_positions=anchor_positions, + ) + loss_per_token_by_position = loss_per_token_by_position.view( + bsz, n_blocks, self.block_size + ) + flat_logits = logits.view(-1, logits.size(-1)) flat_targets = target_ids.view(-1) - ce_per_token = F.cross_entropy(flat_logits, flat_targets, reduction="none") - - loss_per_token = self.ce_loss_alpha * ce_per_token - if self.uses_target_distribution: - if last_hidden_states is None: - raise ValueError( - "DFlash L1 distillation (l1_loss_alpha > 0) requires target " - "last_hidden_states; set inference.store_last_hidden_states=true in the " - "run config." - ) - tgt_idx = (safe_label_indices - 1).clamp(min=0) - aligned_hidden = self._align_target_hidden_states( - last_hidden_states, - tgt_idx, - global_sequence_length=seq_len, - packed_context_positions=packed_context_positions, - ) - target_logits = F.linear(aligned_hidden, lm_head_weight).view(-1, vocab_size) - target_probs = torch.softmax(target_logits.float(), dim=-1) - draft_probs = torch.softmax(flat_logits.float(), dim=-1) - l1_per_token = (draft_probs - target_probs).abs().sum(dim=-1) - loss_per_token = loss_per_token + self.l1_loss_alpha * l1_per_token - - loss_per_token_by_position = ce_per_token.view(bsz, n_blocks, self.block_size) objective_weights = weight_mask if ( diff --git a/aspec/models/domino.py b/aspec/models/domino.py new file mode 100644 index 0000000..acc8d7d --- /dev/null +++ b/aspec/models/domino.py @@ -0,0 +1,135 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from the merged SpecForge Domino training path +# (sgl-project/SpecForge PR #571, specforge/core/domino.py, MIT; see +# licenses/SpecForge-MIT.txt). The objective, predecessor alignment and +# label-shift handling follow that implementation and the public Domino method +# (arXiv:2605.29707). Anchor sampling, block masks, reductions and the DFlash +# training workflow stay in the retained ASpec DFlash path. +"""Domino correction-head training objective on the DFlash block backbone.""" + +import math + +from aspec.models.dflash import DFlashModel + + +class DominoModel(DFlashModel): + """Block-parallel DFlash objective mixed with a causal correction head.""" + + def __init__(self, *args, lambda_base=0.0, **kwargs): + super().__init__(*args, **kwargs) + config = getattr(self.draft_model, "config", None) + if getattr(config, "projector_type", None) != "domino": + raise ValueError("Domino training requires a Domino draft config") + if not hasattr(self.draft_model, "correct_logits"): + raise ValueError("Domino training requires a DominoDraftModel") + if config.block_size != self.block_size: + raise ValueError("Domino training block_size must match its Draft config") + if self.mtp_conditioning is not None: + raise ValueError("Domino does not use the native-MTP conditioning contract") + if self.draft_context_parallel_size != 1: + raise NotImplementedError("Domino currently requires draft_context_parallel_size=1") + if self.loss_objective != "decay": + raise ValueError("Domino supports the decay/uniform weighting objective only") + if self.ce_loss_alpha != 1 or self.l1_loss_alpha != 0: + raise ValueError( + "Domino replaces the CE/L1 mix with its own objective; set " + "ce_loss_alpha=1 and l1_loss_alpha=0" + ) + # Shifted labels start at anchor + 1 and supervise position 0; the + # unshifted layout keeps DFlash's same-position labels and skips the + # anchor slot. Both match the two documented Domino layouts. + self.label_start = 1 if self.draft_model.shift_label else 0 + self.prediction_start = 1 - self.label_start + self.set_lambda_base(lambda_base) + + def _prepare_draft_prefix( + self, + input_ids, + last_hidden_states, + attention_mask, + draft_prefix, + *packed_metadata, + ): + ctx_doc_ids, _, packed_context_positions, packed_document_lengths = packed_metadata + if any( + value is not None + for value in (ctx_doc_ids, packed_context_positions, packed_document_lengths) + ): + raise NotImplementedError( + "Domino supports padded, non-packed inputs only; packed documents " + "or sharded context are rejected before backbone execution" + ) + return super()._prepare_draft_prefix( + input_ids, + last_hidden_states, + attention_mask, + draft_prefix, + *packed_metadata, + ) + + def set_lambda_base(self, lambda_base) -> None: + """Set the backbone-only loss weight for the current step.""" + if ( + isinstance(lambda_base, bool) + or not isinstance(lambda_base, (int, float)) + or not math.isfinite(float(lambda_base)) + ): + raise ValueError("Domino lambda_base must be a finite number in [0, 1]") + lambda_base = float(lambda_base) + if not 0.0 <= lambda_base <= 1.0: + raise ValueError("Domino lambda_base must be a finite number in [0, 1]") + self.lambda_base = lambda_base + + def _training_loss( + self, + *, + logits, + draft_hidden, + target_ids, + last_hidden_states, + lm_head_weight, + safe_label_indices, + packed_context_positions, + sequence_length, + input_ids, + anchor_positions, + ): + base_logits = logits + corrected_logits = self.draft_model.correct_logits( + draft_hidden, base_logits, input_ids, anchor_positions + ) + # The retained CE path supplies the reduction-ready per-token values for + # both the corrected and backbone-only predictions. + _, corrected_per_token, _ = super()._training_loss( + logits=corrected_logits, + draft_hidden=draft_hidden, + target_ids=target_ids, + last_hidden_states=last_hidden_states, + lm_head_weight=lm_head_weight, + safe_label_indices=safe_label_indices, + packed_context_positions=packed_context_positions, + sequence_length=sequence_length, + input_ids=input_ids, + anchor_positions=anchor_positions, + ) + _, base_per_token, _ = super()._training_loss( + logits=base_logits, + draft_hidden=draft_hidden, + target_ids=target_ids, + last_hidden_states=last_hidden_states, + lm_head_weight=lm_head_weight, + safe_label_indices=safe_label_indices, + packed_context_positions=packed_context_positions, + sequence_length=sequence_length, + input_ids=input_ids, + anchor_positions=anchor_positions, + ) + lambda_base = self.lambda_base + loss_per_token = ( + (1.0 - lambda_base) * corrected_per_token + lambda_base * base_per_token + ) + # Report the corrected objective's per-position value; accuracy uses the + # corrected logits as well. + return loss_per_token, corrected_per_token, corrected_logits diff --git a/aspec/models/draft/domino.py b/aspec/models/draft/domino.py new file mode 100644 index 0000000..3e0823b --- /dev/null +++ b/aspec/models/draft/domino.py @@ -0,0 +1,179 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from the merged SpecForge Domino training path +# (sgl-project/SpecForge PR #571, specforge/core/domino.py and the +# modelling/draft/dflash.py projector addition, MIT; see +# licenses/SpecForge-MIT.txt). The GRU-conditioned logit correction, predecessor +# alignment and label-shift conventions follow that implementation and the +# public Domino method (arXiv:2605.29707). ASpec supplies its own configuration, +# fail-closed draft contract and DFlash training integration. +"""Domino Draft: block-parallel DFlash backbone with a causal correction head.""" + +import torch +from torch import nn + +from aspec.models.draft.dflash import DFlashConfig, DFlashDraftModel + + +class DominoConfig(DFlashConfig): + """DFlash block backbone plus a GRU-conditioned correction head.""" + + model_type = "domino" + + def __init__( + self, + *, + projector_type="domino", + emb_dim=256, + gru_hidden_dim=1024, + shift_label=True, + pure_draft_prefix_len=1, + block_size=16, + **kwargs, + ): + if projector_type != "domino": + raise ValueError("Domino requires projector_type='domino'") + for name, value in (("emb_dim", emb_dim), ("gru_hidden_dim", gru_hidden_dim)): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"Domino requires a positive integer {name}") + if not isinstance(shift_label, bool): + raise ValueError("Domino shift_label must be a boolean") + if isinstance(pure_draft_prefix_len, bool) or not isinstance( + pure_draft_prefix_len, int + ): + raise ValueError("Domino pure_draft_prefix_len must be an integer") + if pure_draft_prefix_len < 0: + raise ValueError("Domino pure_draft_prefix_len must be non-negative") + if isinstance(block_size, bool) or not isinstance(block_size, int) or block_size < 2: + raise ValueError("Domino block_size must be an integer >= 2") + suffix_start = pure_draft_prefix_len if shift_label else 1 + pure_draft_prefix_len + if not 0 <= suffix_start < block_size: + raise ValueError( + "Domino pure_draft_prefix_len leaves no corrected position in the block" + ) + architectures = kwargs.pop("architectures", ["DominoDraftModel"]) + if architectures != ["DominoDraftModel"]: + raise ValueError("Domino uses exactly one DominoDraftModel architecture") + if kwargs.get("is_causal"): + raise ValueError( + "Domino keeps the block-parallel DFlash backbone; causal modelling " + "belongs to the correction head" + ) + kwargs.update( + projector_type="domino", + emb_dim=emb_dim, + gru_hidden_dim=gru_hidden_dim, + shift_label=shift_label, + pure_draft_prefix_len=pure_draft_prefix_len, + ) + super().__init__( + architectures=architectures, block_size=block_size, **kwargs + ) + if self.ffn_type != "dense": + raise ValueError("Domino currently supports dense Draft FFNs only") + layer_types = list(getattr(self, "layer_types", []) or []) + if any(kind != "full_attention" for kind in layer_types): + raise ValueError("Domino currently supports full intra-block attention only") + if getattr(self, "mtp_conditioning", None) is not None: + raise ValueError("Domino does not use the native-MTP conditioning contract") + for stage in ("csa_training_stage", "qsa_training_stage", "dsa_training_stage"): + if getattr(self, stage, None) is not None: + raise ValueError("Domino is not integrated with sparse-attention training") + + +class DominoDraftModel(DFlashDraftModel): + """DFlash block backbone with the Domino GRU correction head.""" + + config_class = DominoConfig + + def __init__(self, config): + super().__init__(config) + self.projector_type = config.projector_type + self.shift_label = config.shift_label + self.pure_draft_prefix_len = config.pure_draft_prefix_len + self.suffix_start = ( + config.pure_draft_prefix_len + if config.shift_label + else 1 + config.pure_draft_prefix_len + ) + self.emb_dim = config.emb_dim + self.gru_hidden_dim = config.gru_hidden_dim + # Reference names are retained so future export can map checkpoint keys. + self.prefix_gru = nn.GRU( + input_size=config.hidden_size, + hidden_size=config.gru_hidden_dim, + num_layers=1, + batch_first=True, + bias=False, + ) + self.embed_proj = nn.Sequential( + nn.Linear( + config.hidden_size + config.gru_hidden_dim, + config.emb_dim, + bias=False, + ), + nn.SiLU(), + nn.Linear(config.emb_dim, config.vocab_size, bias=False), + ) + + def correct_logits( + self, + draft_hidden: torch.Tensor, + base_logits: torch.Tensor, + input_ids: torch.Tensor, + anchor_positions: torch.Tensor, + ) -> torch.Tensor: + """Add the causal correction delta to the block suffix. + + ``draft_hidden`` and ``base_logits`` are ``[B, N * block_size, ...]``. + Position 0 (or ``suffix_start`` for a pure draft prefix) keeps the + backbone logits; every later position receives the GRU-conditioned + delta. ``shift_label`` selects the reference alignment: the shifted + layout consumes ``anchor ... anchor + block_size - 1`` tokens, while the + unshifted layout consumes the first ``block_size - 1`` of them. + """ + batch, draft_len = base_logits.shape[:2] + block_size = self.config.block_size + if draft_len != anchor_positions.shape[1] * block_size: + raise ValueError("Domino logits do not match the anchor/block layout") + hidden = draft_hidden.reshape(batch, -1, block_size, self.hidden_size) + logits = base_logits.reshape(batch, -1, block_size, base_logits.size(-1)) + + offsets = torch.arange(block_size, device=input_ids.device).view(1, 1, -1) + token_indices = (anchor_positions.unsqueeze(-1) + offsets).clamp( + max=input_ids.size(1) - 1 + ) + block_ids = torch.gather( + input_ids.unsqueeze(1).expand(-1, anchor_positions.shape[1], -1), + 2, + token_indices, + ) + + if self.shift_label: + gru_inputs = block_ids + state_slice = slice(self.suffix_start, None) + else: + gru_inputs = block_ids[..., : block_size - 1] + state_slice = slice(self.suffix_start - 1, None) + + block_emb = self.embed_tokens(gru_inputs).to(self.prefix_gru.weight_ih_l0.dtype) + states, _ = self.prefix_gru( + block_emb.reshape(batch * anchor_positions.shape[1], -1, self.hidden_size) + ) + states = states.reshape( + batch, anchor_positions.shape[1], gru_inputs.size(-1), self.gru_hidden_dim + )[:, :, state_slice, :] + suffix_hidden = hidden[:, :, self.suffix_start :, :] + if states.size(2) != suffix_hidden.size(2): + raise ValueError("Domino correction state and hidden suffix lengths differ") + + delta = self.embed_proj(torch.cat([suffix_hidden, states], dim=-1)) + corrected = torch.cat( + [ + logits[:, :, : self.suffix_start, :], + logits[:, :, self.suffix_start :, :] + delta.to(logits.dtype), + ], + dim=2, + ) + return corrected.reshape(batch, draft_len, logits.size(-1)) diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index c2eece3..d61105b 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -22,6 +22,8 @@ the retained source identities; applicable license and notice texts are distribu | [SpecForge DMTP capture](https://github.com/sgl-project/SpecForge/tree/c02184448f9b000060df50307b7044c3b475ad79) | Inspiration for Target-side frozen MTP feature production; ASpec adapts its own native predictor and publication/transport lifecycle, without copying the capture plugin | | [Speculators](https://github.com/vllm-project/speculators) `7ec7aba9450b533334e8f7992baf4e93be3c1a57` | Native MTP recurrent objective and checkpoint bindings; ASpec supplies its own training/layout integration | | [Speculators](https://github.com/vllm-project/speculators) `3702320434bbdaa853f0846a66e6b6b8f1d97aec` | Adapted P-EAGLE model/loss and Qwen3 first-layer fusion; deterministic dense layout and ASpec input/observation boundary | +| [Domino](https://github.com/jianuo-huang/Domino) `44e1e6912ba9d301899739a54bfda2ce8826980a` and [arXiv:2605.29707](https://arxiv.org/abs/2605.29707) | Public method and label-layout reference for the GRU correction head; independently implemented on the retained DFlash block backbone | +| [SpecForge Domino](https://github.com/sgl-project/SpecForge/pull/571) merge `7de39e32fc845caf59bca9841595145d24e1796f` | Adapted Domino objective, predecessor alignment and label-shift handling (MIT); ASpec supplies configuration, fail-closed scope and DFlash integration | | [Ling-3.0-flash](https://huggingface.co/inclusionAI/Ling-3.0-flash) `fcc60a52b34c69c6271d0ac8a8cb0815020b6a43` | Adapted MoE routing and independently retained reference class bodies; Apache-2.0 source headers preserved | | [EAGLE](https://github.com/SafeAILab/EAGLE), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) and Transformers through TorchSpec | Model/data helpers in `aspec/models/draft/base.py`, shared checkpoint reads in `aspec/models/checkpoint.py`, and `aspec/data/preprocessing.py`; original source headers retain the transitive attribution | diff --git a/docs/support.md b/docs/support.md index fe553a6..1c01a98 100644 --- a/docs/support.md +++ b/docs/support.md @@ -12,6 +12,7 @@ and tests do not imply CPU training support. | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | +| Domino core | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts, eager/SDPA | Model/loss prerequisite only; dense full attention, CP1 padded execution; no trainer, curriculum schedule, resume, export or consumer validation; see [scope](user-guide/domino.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | MTP-conditioned DSpark / DFlash | Target-side frozen native Qwen3.5 features, token/hidden dual-anchor training, exact recovery and strict native assets; legacy Draft-side assets retained | Dense full GQA, CP1, MBS1, REPLICATE, one node, non-packing, decay objective; Target capture PP1/unquantized; no generic serving consumer | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 9ebe89d..b895b63 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -61,6 +61,16 @@ The [P-EAGLE model/loss API](peagle.md) provides a dense unsampled Qwen3 paralle Draft and forward-KL objective with independent numerical checks. Public training, COD sampling, recovery, export and consumer validation remain pending. +## Domino core prerequisite + +The [Domino model/loss API](domino.md) adds a GRU-conditioned causal correction head +on the retained DFlash block backbone and mixes corrected with backbone-only +cross-entropy through `lambda_base`. Both documented label layouts are explicit: +the unshifted DFlash convention and the official shifted convention +(`shift_label=true`, `pure_draft_prefix_len=1`). Dense full attention is enforced. +Public training selection, the `lambda_base` curriculum, recovery, export and +consumer validation remain pending. + ## EAGLE4: two stages The Draft stage trains the full recurrent model. The candidate stage freezes the entire Draft, diff --git a/docs/user-guide/domino.md b/docs/user-guide/domino.md new file mode 100644 index 0000000..1114af4 --- /dev/null +++ b/docs/user-guide/domino.md @@ -0,0 +1,124 @@ +# Domino model and loss core + +This is the first prerequisite for [ASpec #38](https://github.com/inclusionAI/ASpec/issues/38). +It provides `aspec.models.draft.domino.DominoDraftModel` and +`aspec.models.domino.DominoModel`: a block-parallel DFlash backbone with a +GRU-conditioned causal correction head. It is a model/loss API; the ASpec training +CLI does not select this algorithm yet. + +## Reference and selected scope + +The implementation is **adapted** from the merged +[SpecForge Domino training path](https://github.com/sgl-project/SpecForge/pull/571) +(`specforge/core/domino.py` and the DFlash projector addition, merge +`7de39e32fc845caf59bca9841595145d24e1796f`, MIT; see +`licenses/SpecForge-MIT.txt`) and follows the public +[Domino method](https://arxiv.org/abs/2605.29707) at +[jianuo-huang/Domino @ `44e1e6912ba9d301899739a54bfda2ce8826980a`](https://github.com/jianuo-huang/Domino/tree/44e1e6912ba9d301899739a54bfda2ce8826980a) +for the architecture and label layout. It is not a reproduction of the paper's +training data, budget or acceptance results, and it does not copy the unlicensed +reference repository's source. + +The selected first scope is one dense full-attention DFlash backbone such as Qwen3 +with a frozen Target LM head and features. Dense FFNs and full intra-block attention +are enforced. Native MTP conditioning, sparse-attention training stages and MoE +FFNs fail closed. + +## Correction head + +The head owns two parameter groups, named as in the reference for future export: + +- `prefix_gru`: a bias-free GRU over the block's real tokens, producing a causal + predecessor state per suffix position; +- `embed_proj`: `Linear(hidden + gru_hidden) -> SiLU -> Linear(vocab)` producing the + logit delta. + +The block backbone itself stays block-parallel: attention visibility, anchor +sampling, noise construction and Target-feature fusion are unchanged from DFlash. +Only the correction head is causal, and its inputs are the **ground-truth block +tokens**, not Target features. + +## Label-shift layouts + +`DominoConfig` makes the alignment explicit instead of assuming one checkpoint +layout. `shift_label` and `pure_draft_prefix_len` select the reference conventions: + +| Field | `shift_label=true` (official Qwen3) | `shift_label=false` | +| --- | --- | --- | +| Supervised label for block position `k` | `x[anchor + 1 + k]` | `x[anchor + k]` | +| First supervised block position | `0` | `1` (anchor is context only) | +| `suffix_start` | `pure_draft_prefix_len` | `1 + pure_draft_prefix_len` | +| GRU input tokens | `x[anchor .. anchor + block_size - 1]`, states used from `suffix_start` | `x[anchor .. anchor + block_size - 2]`, states used from `suffix_start - 1` | + +`pure_draft_prefix_len` must leave at least one corrected position in the block. +The official `Qwen3-8B-Domino-b16` configuration uses `shift_label=true` and +`pure_draft_prefix_len=1`, so block position 0 keeps the backbone logits and +positions 1..`block_size-1` receive the GRU delta. The unshifted layout matches +plain DFlash supervision: position 0 is the anchor and excluded from the loss. + +## Objective + +`DominoModel` mixes the corrected and backbone-only cross-entropies: + +```text +loss = (1 - lambda_base) * CE(corrected logits, labels) + + lambda_base * CE(backbone logits, labels) +``` + +`lambda_base` is an explicit per-step value in `[0, 1]`; use +`model.set_lambda_base(value)` from a trainer-owned curriculum. `lambda_base=1` +removes the correction contribution and keeps the selected label layout (the +correction head receives no useful gradient). It equals the plain DFlash objective +only for the unshifted layout (`shift_label=false`) with otherwise matching +weighting settings; the shifted layout supervises different slots. Anchor sampling, +block masks, label bounds, `loss_mask` and DFlash-style exponential depth decay are +reused; `ce_loss_alpha=1` and `l1_loss_alpha=0` are enforced. Reporting accuracy and per-position loss use the +corrected logits, while the returned loss applies the DFlash objective weights. + +## Minimal numerical use + +Inside the [isolated Linux environment](installation.md): + +```python +import torch +from aspec.models.domino import DominoModel +from aspec.models.draft.domino import DominoConfig, DominoDraftModel + +config = DominoConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, emb_dim=6, gru_hidden_dim=8, + shift_label=True, pure_draft_prefix_len=1, +) +draft = DominoDraftModel(config) +draft.embed_tokens.requires_grad_(False) +model = DominoModel( + draft, block_size=5, num_anchors=2, attention_backend="sdpa", + lambda_base=0.3, +) +result = model( + input_ids=torch.arange(12).reshape(1, 12) % 32, + hidden_states_list=torch.randn(1, 12, 16), + loss_mask=torch.ones(1, 12), + lm_head_weight=torch.randn(32, 16), + last_hidden_states=torch.randn(1, 12, 16), + injected_anchors=torch.tensor([[2, 7]]), + injected_keep_mask=torch.ones(1, 2, dtype=torch.bool), +) +result[0].backward() +``` + +Run `HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 python -m pytest -q tests/cpu/test_domino.py` +for configuration fail-closed checks, independent forward/gradient equations for both +label layouts, `lambda_base` identity, anchor exclusion, loss-mask and short-sequence +handling, empty supervision, and frozen Target/embedding behavior. + +## Remaining work + +Trainer/Plan selection, the `lambda_base` curriculum schedule, a runnable recipe, +exact resume of backbone/head/curriculum state, native export with strict reloading, +Target bindings, proposal consumers and held-out backbone-vs-corrected observations +remain pending. CPU numerical checks do not establish Qwen3-8B training quality, +GPU throughput or decoding acceptance. diff --git a/tests/cpu/test_domino.py b/tests/cpu/test_domino.py new file mode 100644 index 0000000..bc71bdb --- /dev/null +++ b/tests/cpu/test_domino.py @@ -0,0 +1,540 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Domino causal correction head and mixed backbone/head objective. + +Reference: Domino, arXiv:2605.29707, and the merged SpecForge training path +(PR #571). These checks compare the implementation with explicit equations; +they do not reproduce the paper's training data, budget or serving results. +""" + +import copy + +import pytest +import torch +from torch.nn import functional as F + +from aspec.models.dflash import DFlashModel +from aspec.models.domino import DominoModel +from aspec.models.draft.dflash import DFlashConfig, DFlashDraftModel +from aspec.models.draft.domino import DominoConfig, DominoDraftModel + + +def make_config(**overrides): + values = dict( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, rms_norm_eps=1e-6, emb_dim=6, gru_hidden_dim=8, + ) + values.update(overrides) + return DominoConfig(**values) + + +def make_model( + *, shift_label=True, pure_draft_prefix_len=None, lambda_base=0.3, + loss_decay_gamma=0.0, seed=17, **config_overrides +): + if pure_draft_prefix_len is None: + pure_draft_prefix_len = 1 if shift_label else 0 + with torch.random.fork_rng(devices=[]): + torch.random.default_generator.manual_seed(seed) + draft = DominoDraftModel( + make_config( + shift_label=shift_label, + pure_draft_prefix_len=pure_draft_prefix_len, + **config_overrides, + ) + ) + draft.embed_tokens.requires_grad_(False) + return DominoModel( + draft, block_size=draft.config.block_size, num_anchors=2, + attention_backend="sdpa", loss_decay_gamma=loss_decay_gamma, + lambda_base=lambda_base, + ) + + +def make_inputs(seed=31, length=12, batch=1): + generator = torch.Generator().manual_seed(seed) + ids = torch.arange(batch * length).reshape(batch, length) % 32 + return dict( + input_ids=ids, + hidden_states_list=torch.randn(batch, length, 16, generator=generator), + loss_mask=torch.ones_like(ids), + lm_head_weight=torch.randn(32, 16, generator=generator), + last_hidden_states=torch.randn(batch, length, 16, generator=generator), + injected_anchors=torch.tensor([[2, 7]]).expand(batch, -1), + injected_keep_mask=torch.ones(batch, 2, dtype=torch.bool), + ) + + +def run_and_capture(model, inputs): + captured = [] + handle = model.draft_model.register_forward_hook( + lambda _, __, output: captured.append(output) + ) + result = model(**inputs) + handle.remove() + return result, captured[0] + + +def reference_objective(model, draft_hidden, inputs): + """Dense Domino equations with explicit label/anchor/mask alignment.""" + draft_model = model.draft_model + anchors = inputs["injected_anchors"] + keep = inputs["injected_keep_mask"] + ids = inputs["input_ids"] + block_size = model.block_size + suffix = draft_model.suffix_start + + offsets = torch.arange(model.label_start, model.label_start + block_size)[None, None] + label_indices = anchors.unsqueeze(-1) + offsets + valid = label_indices < ids.shape[1] + safe_indices = label_indices.clamp(max=ids.shape[1] - 1) + expanded_ids = ids.unsqueeze(1).expand(-1, anchors.shape[1], -1) + target_ids = torch.gather(expanded_ids, 2, safe_indices) + + base_logits = F.linear(draft_hidden, inputs["lm_head_weight"]) + hidden4d = draft_hidden.reshape(-1, anchors.shape[1], block_size, 16) + logits4d = base_logits.reshape(-1, anchors.shape[1], block_size, 32) + + block_offsets = torch.arange(block_size)[None, None] + block_ids = torch.gather( + expanded_ids, + 2, + (anchors.unsqueeze(-1) + block_offsets).clamp(max=ids.shape[1] - 1), + ) + if draft_model.shift_label: + gru_inputs = block_ids + state_slice = slice(suffix, None) + else: + gru_inputs = block_ids[..., : block_size - 1] + state_slice = slice(suffix - 1, None) + states, _ = draft_model.prefix_gru( + draft_model.embed_tokens(gru_inputs).reshape( + -1, gru_inputs.size(-1), draft_model.hidden_size + ) + ) + states = states.reshape(-1, anchors.shape[1], gru_inputs.size(-1), 8)[ + :, :, state_slice, : + ] + delta = draft_model.embed_proj(torch.cat([hidden4d[:, :, suffix:, :], states], dim=-1)) + corrected = torch.cat( + [logits4d[:, :, :suffix, :], logits4d[:, :, suffix:, :] + delta], dim=2 + ) + corrected_ce = F.cross_entropy( + corrected.reshape(-1, 32), target_ids.reshape(-1), reduction="none" + ) + base_ce = F.cross_entropy( + base_logits.reshape(-1, 32), target_ids.reshape(-1), reduction="none" + ) + + gathered_mask = torch.gather( + inputs["loss_mask"].unsqueeze(1).expand(-1, anchors.shape[1], -1), + 2, safe_indices, + ) + positions = torch.arange(block_size)[None, None] + weights = keep.unsqueeze(-1).float() * valid.float() + weights = weights * (positions >= model.prediction_start).float() + weights = weights * gathered_mask + decayed = weights + if model.loss_decay_gamma is not None and model.loss_decay_gamma > 0: + decayed = weights * torch.exp( + -(positions - model.prediction_start).clamp_min(0).float() + / model.loss_decay_gamma + ) + flat_weights = decayed.reshape(-1) + mixed = (1.0 - model.lambda_base) * corrected_ce + model.lambda_base * base_ce + loss = (mixed * flat_weights).sum() / flat_weights.sum().clamp_min(1e-6) + + binary = weights.reshape(-1) + count = binary.reshape(-1, anchors.shape[1], block_size).sum(dim=(0, 1)) + per_position = ( + corrected_ce.reshape(-1, anchors.shape[1], block_size) * weights + ).sum(dim=(0, 1)) / count.clamp(min=1.0) + return loss, corrected_ce.reshape(-1, anchors.shape[1], block_size), per_position + + +def _assert_parameter_gradients(actual, expected): + actual_parameters = dict(actual.named_parameters()) + expected_parameters = dict(expected.named_parameters()) + assert actual_parameters.keys() == expected_parameters.keys() + for name, parameter in actual_parameters.items(): + reference = expected_parameters[name] + assert (parameter.grad is None) == (reference.grad is None), name + if parameter.grad is not None: + assert torch.isfinite(parameter.grad).all(), name + torch.testing.assert_close( + parameter.grad, reference.grad, rtol=2e-5, atol=2e-6, msg=name + ) + + +def test_config_pins_the_correction_contract(): + config = make_config() + assert config.model_type == "domino" + assert config.architectures == ["DominoDraftModel"] + assert config.projector_type == "domino" + assert config.shift_label is True + assert config.pure_draft_prefix_len == 1 + restored = DominoConfig.from_dict(config.to_dict()) + assert restored.architectures == ["DominoDraftModel"] + assert restored.shift_label is True + assert restored.pure_draft_prefix_len == 1 + assert restored.emb_dim == 6 and restored.gru_hidden_dim == 8 + + +@pytest.mark.parametrize("override", [ + {"projector_type": "linear"}, + {"emb_dim": 0}, + {"emb_dim": 6.5}, + {"gru_hidden_dim": -1}, + {"shift_label": 1}, + {"pure_draft_prefix_len": 1.5}, + {"pure_draft_prefix_len": -1}, + {"shift_label": False, "pure_draft_prefix_len": -1}, + {"shift_label": False, "pure_draft_prefix_len": 4}, + {"architectures": ["DFlashDraftModel"]}, + {"is_causal": True}, +]) +def test_config_fail_closed(override): + with pytest.raises(ValueError): + make_config(**override) + + +def test_config_rejects_unsupported_backbones(): + with pytest.raises(ValueError): + make_config(ffn_type="moe", num_experts=4, num_experts_per_tok=2, + moe_intermediate_size=32) + with pytest.raises(ValueError): + make_config(layer_types=["sliding_attention"], sliding_window=64) + + +def test_shifted_layout_supervises_the_anchor_slot(): + shifted = make_model(shift_label=True) + assert shifted.label_start == 1 + assert shifted.prediction_start == 0 + result, _ = run_and_capture(shifted, make_inputs()) + assert int(result[4][0]) > 0 + + unshifted = make_model(shift_label=False) + assert unshifted.label_start == 0 + assert unshifted.prediction_start == 1 + result, _ = run_and_capture(unshifted, make_inputs()) + assert int(result[4][0]) == 0 + + +@pytest.mark.parametrize("shift_label,pure", [(True, 1), (False, 0), (True, 0), (False, 1)]) +@pytest.mark.parametrize("loss_decay_gamma", [0.0, 3.0]) +@pytest.mark.parametrize("lambda_base", [0.0, 0.3, 1.0]) +def test_loss_and_gradients_match_independent_equations( + shift_label, pure, loss_decay_gamma, lambda_base +): + actual = make_model( + shift_label=shift_label, pure_draft_prefix_len=pure, + lambda_base=lambda_base, loss_decay_gamma=loss_decay_gamma, + ) + reference = copy.deepcopy(actual) + inputs, reference_inputs = make_inputs(), make_inputs() + + result, captured = run_and_capture(actual, inputs) + _, reference_captured = run_and_capture(reference, reference_inputs) + expected_loss, expected_ce, expected_per_position = reference_objective( + reference, reference_captured, reference_inputs + ) + actual_loss, actual_ce, actual_per_position = reference_objective( + actual, captured, inputs + ) + + torch.testing.assert_close(result[0], expected_loss, rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_loss, expected_loss, rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_ce, expected_ce, rtol=2e-5, atol=2e-6) + torch.testing.assert_close(result[2], expected_per_position, rtol=2e-5, atol=2e-6) + assert actual_per_position.shape == (actual.block_size,) + + result[0].backward() + expected_loss.backward() + _assert_parameter_gradients(actual, reference) + + +def test_lambda_one_recovers_the_backbone_only_objective(): + model = make_model(lambda_base=1.0, loss_decay_gamma=0.0) + inputs = make_inputs() + result, captured = run_and_capture(model, inputs) + expected, _, _ = reference_objective(model, captured, inputs) + torch.testing.assert_close(result[0], expected, rtol=2e-5, atol=2e-6) + result[0].backward() + for parameter in ( + model.draft_model.prefix_gru.weight_ih_l0, + model.draft_model.embed_proj[0].weight, + model.draft_model.embed_proj[2].weight, + ): + assert parameter.grad is None or torch.count_nonzero(parameter.grad) == 0 + assert torch.count_nonzero(model.draft_model.context_proj.weight.grad) > 0 + + +def test_two_row_layout_matches_reference_and_backward(): + actual = make_model(shift_label=True, lambda_base=0.4) + reference = copy.deepcopy(actual) + inputs, reference_inputs = make_inputs(batch=2), make_inputs(batch=2) + result, captured = run_and_capture(actual, inputs) + _, reference_captured = run_and_capture(reference, reference_inputs) + expected_loss, expected_ce, _ = reference_objective( + reference, reference_captured, reference_inputs + ) + actual_loss, actual_ce, _ = reference_objective(actual, captured, inputs) + assert result[0].ndim == 0 + torch.testing.assert_close(result[0], expected_loss, rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_loss, expected_loss, rtol=2e-5, atol=2e-6) + torch.testing.assert_close(actual_ce, expected_ce, rtol=2e-5, atol=2e-6) + result[0].backward() + expected_loss.backward() + _assert_parameter_gradients(actual, reference) + + +def test_loss_mask_excludes_unsupervised_labels(): + model = make_model(shift_label=True) + inputs = make_inputs() + inputs["loss_mask"][0, 6] = 0 + result, captured = run_and_capture(model, inputs) + expected, _, _ = reference_objective(model, captured, inputs) + torch.testing.assert_close(result[0], expected, rtol=2e-5, atol=2e-6) + offsets = torch.arange(model.label_start, model.label_start + model.block_size) + label_indices = inputs["injected_anchors"].unsqueeze(-1) + offsets + gathered = torch.gather( + inputs["loss_mask"].unsqueeze(1).expand(-1, 2, -1), + 2, label_indices.clamp(max=inputs["input_ids"].size(1) - 1), + ) + valid = (label_indices < inputs["input_ids"].size(1)) & (gathered > 0.5) + valid = valid & (torch.arange(model.block_size) >= model.prediction_start) + assert int(result[4].sum()) == int(valid.sum()) + + +def test_empty_supervision_is_neutral_and_finite(): + model = make_model() + inputs = make_inputs() + inputs["loss_mask"] = torch.zeros_like(inputs["loss_mask"]) + inputs["injected_keep_mask"] = torch.zeros_like(inputs["injected_keep_mask"]) + result, _ = run_and_capture(model, inputs) + assert float(result[0].detach()) == 0.0 + assert torch.isfinite(result[0]) + assert int(result[4].sum()) == 0 + result[0].backward() + for parameter in model.parameters(): + assert parameter.grad is None or torch.count_nonzero(parameter.grad) == 0 + + +def test_target_features_are_detached_and_embedding_stays_frozen(): + model = make_model() + inputs = make_inputs() + inputs["last_hidden_states"].requires_grad_(True) + result, _ = run_and_capture(model, inputs) + result[0].backward() + assert inputs["last_hidden_states"].grad is None + assert model.draft_model.embed_tokens.weight.grad is None + + +@pytest.mark.parametrize("length", [3, 6]) +def test_short_sequences_keep_loss_finite(length): + model = make_model() + inputs = make_inputs(length=length) + inputs["injected_anchors"] = torch.tensor([[0, max(length - 3, 1)]]) + result, captured = run_and_capture(model, inputs) + expected, _, _ = reference_objective(model, captured, inputs) + torch.testing.assert_close(result[0], expected, rtol=2e-5, atol=2e-6) + assert torch.isfinite(result[0]) + result[0].backward() + for parameter in model.parameters(): + if parameter.grad is not None: + assert torch.isfinite(parameter.grad).all() + + +@pytest.mark.parametrize("lambda_base", [-0.1, 1.1, float("nan"), float("inf"), True]) +def test_invalid_lambda_base_fails_closed(lambda_base): + draft = DominoDraftModel(make_config()) + with pytest.raises(ValueError, match="lambda_base"): + DominoModel( + draft, block_size=5, num_anchors=2, attention_backend="sdpa", + lambda_base=lambda_base, + ) + + +def test_set_lambda_base_validates_and_changes_the_objective(): + model = make_model(lambda_base=0.0) + model.set_lambda_base(1.0) + assert model.lambda_base == 1.0 + with pytest.raises(ValueError, match="lambda_base"): + model.set_lambda_base(float("nan")) + with pytest.raises(ValueError, match="lambda_base"): + model.set_lambda_base(2.0) + + +def test_non_domino_inputs_fail_closed(): + config = DFlashConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, + ) + draft = DFlashDraftModel(config) + with pytest.raises(ValueError, match="Domino draft config"): + DominoModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa") + with pytest.raises(ValueError, match="block_size"): + DominoModel( + DominoDraftModel(make_config()), block_size=4, num_anchors=2, + attention_backend="sdpa", + ) + with pytest.raises(ValueError, match="CE/L1"): + DominoModel( + DominoDraftModel(make_config()), block_size=5, num_anchors=2, + attention_backend="sdpa", ce_loss_alpha=0.5, + ) + with pytest.raises(ValueError, match="CE/L1"): + DominoModel( + DominoDraftModel(make_config()), block_size=5, num_anchors=2, + attention_backend="sdpa", l1_loss_alpha=1.0, + ) + + +def test_correction_keeps_backbone_only_prefix_and_changes_the_suffix(): + base = make_model(shift_label=True) + corrected = copy.deepcopy(base) + inputs = make_inputs() + inputs["injected_anchors"] = torch.tensor([[2, 7]]) + _, base_hidden = run_and_capture(base, inputs) + _, corrected_hidden = run_and_capture(corrected, inputs) + torch.testing.assert_close(base_hidden, corrected_hidden, rtol=0, atol=0) + + base_logits = F.linear(base_hidden, inputs["lm_head_weight"]) + corrected_logits = base.draft_model.correct_logits( + base_hidden, base_logits, inputs["input_ids"], inputs["injected_anchors"] + ) + base4d = base_logits.reshape(1, 2, 5, 32) + corrected4d = corrected_logits.reshape(1, 2, 5, 32) + suffix = base.draft_model.suffix_start + torch.testing.assert_close( + base4d[:, :, :suffix], corrected4d[:, :, :suffix], rtol=0, atol=0 + ) + assert not torch.equal(base4d[:, :, suffix:], corrected4d[:, :, suffix:]) + + +def test_plain_dflash_wrapper_keeps_unshifted_layout(): + config = DFlashConfig( + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + num_target_layers=1, target_hidden_size=16, target_num_hidden_layers=4, + target_layer_ids=[1], mask_token_id=31, max_position_embeddings=128, + block_size=5, + ) + draft = DFlashDraftModel(config) + model = DFlashModel(draft, block_size=5, num_anchors=2, attention_backend="sdpa") + assert model.label_start == 0 + result, _ = run_and_capture(model, make_inputs()) + assert int(result[4][0]) == 0 + + +def _plain_dflash_with_domino_backbone(domino): + draft = domino.draft_model + config = DFlashConfig( + hidden_size=draft.config.hidden_size, + intermediate_size=draft.config.intermediate_size, + num_hidden_layers=draft.config.num_hidden_layers, + num_attention_heads=draft.config.num_attention_heads, + num_key_value_heads=draft.config.num_key_value_heads, + head_dim=draft.config.head_dim, + vocab_size=draft.config.vocab_size, + num_target_layers=draft.config.num_target_layers, + target_hidden_size=draft.config.target_hidden_size, + target_num_hidden_layers=draft.config.target_num_hidden_layers, + target_layer_ids=list(draft.config.target_layer_ids), + mask_token_id=draft.config.mask_token_id, + max_position_embeddings=draft.config.max_position_embeddings, + block_size=draft.config.block_size, + rms_norm_eps=draft.config.rms_norm_eps, + ) + plain_draft = DFlashDraftModel(config) + backbone = { + name: value + for name, value in draft.state_dict().items() + if not name.startswith(("prefix_gru.", "embed_proj.")) + } + plain_draft.load_state_dict(backbone, strict=True) + plain_draft.embed_tokens.requires_grad_(False) + return DFlashModel( + plain_draft, + block_size=domino.block_size, + num_anchors=domino.num_anchors, + attention_backend="sdpa", + loss_decay_gamma=domino.loss_decay_gamma, + ) + + +@pytest.mark.parametrize( + "field", ["ctx_doc_ids", "packed_context_positions", "packed_document_lengths"] +) +def test_packed_inputs_are_rejected_before_backbone(field): + model = make_model(shift_label=True) + inputs = make_inputs() + if field == "packed_document_lengths": + inputs[field] = (inputs["input_ids"].size(1),) + else: + inputs[field] = torch.zeros_like(inputs["input_ids"]) + with pytest.raises(NotImplementedError, match="packed"): + model(**inputs) + + +def test_unshifted_lambda_one_matches_plain_dflash_loss_and_gradients(): + domino = make_model(shift_label=False, lambda_base=1.0, loss_decay_gamma=0.0) + plain = _plain_dflash_with_domino_backbone(domino) + inputs = make_inputs() + domino_result, _ = run_and_capture(domino, inputs) + plain_result, _ = run_and_capture(plain, inputs) + torch.testing.assert_close( + domino_result[0], plain_result[0], rtol=2e-5, atol=2e-6 + ) + + domino_result[0].backward() + plain_result[0].backward() + domino_parameters = dict(domino.named_parameters()) + for name, parameter in plain.named_parameters(): + actual = domino_parameters[name] + assert (actual.grad is None) == (parameter.grad is None), name + if parameter.grad is not None: + torch.testing.assert_close( + actual.grad, parameter.grad, rtol=2e-5, atol=2e-6, msg=name + ) + for name in ( + "draft_model.prefix_gru.weight_ih_l0", + "draft_model.embed_proj.0.weight", + "draft_model.embed_proj.2.weight", + ): + grad = domino_parameters[name].grad + assert grad is None or torch.count_nonzero(grad) == 0, name + + +def test_correction_ignores_future_block_tokens(): + model = make_model(shift_label=True) + draft = model.draft_model + block_size = model.block_size + blocks = 2 + generator = torch.Generator().manual_seed(5) + hidden = torch.randn( + 1, blocks * block_size, draft.hidden_size, generator=generator + ) + base_logits = torch.randn( + 1, blocks * block_size, draft.config.vocab_size, generator=generator + ) + input_ids = ( + torch.arange(blocks * block_size, dtype=torch.long).reshape(1, -1) + % draft.config.vocab_size + ) + anchors = torch.tensor([[0, block_size]]) + corrected = draft.correct_logits(hidden, base_logits, input_ids, anchors) + + future = input_ids.clone() + future[0, -1] = (future[0, -1] + 1) % draft.config.vocab_size + changed = draft.correct_logits(hidden, base_logits, future, anchors) + + torch.testing.assert_close(corrected[:, :-1], changed[:, :-1], rtol=0, atol=0) + assert not torch.equal(corrected[:, -1:], changed[:, -1:])