From 5e45039b041a03af8576d1f3405f4e541a65f745 Mon Sep 17 00:00:00 2001 From: Longxmas <1185267696@qq.com> Date: Tue, 17 Nov 2026 18:23:49 +0800 Subject: [PATCH] feat(dspark): support FP8 frozen LM heads in regular training Select BF16 or rowwise E4M3 for the DSpark draft projection through the training recipe. Rebuild forward and input-gradient caches from canonical weights on initialization and exact resume. Share the operator with the resident benchmark and optional SGLang draft adapter. --- aspec/config/train_config.py | 2 + aspec/inference/dspark_fp8_head.py | 182 +++++++++++++++++++++ aspec/models/dspark.py | 22 ++- aspec/models/ops/dspark_fp8_lm_head.py | 191 +++++++++++++++++++++++ aspec/planning.py | 12 ++ aspec/training/checkpoint.py | 3 + aspec/training/dspark_trainer.py | 14 ++ configs/recipes/qwen3_8b_dspark.yaml | 1 + tests/cpu/test_planning.py | 36 +++++ tests/test_fp8_dspark_inference.py | 118 ++++++++++++++ tests/test_fp8_lm_head.py | 182 +++++++++++++++++++++ tools/performance/torchspec_benchmark.py | 5 + 12 files changed, 760 insertions(+), 8 deletions(-) create mode 100644 aspec/inference/dspark_fp8_head.py create mode 100644 aspec/models/ops/dspark_fp8_lm_head.py create mode 100644 tests/test_fp8_dspark_inference.py create mode 100644 tests/test_fp8_lm_head.py diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index efb4107..e701009 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -246,6 +246,8 @@ class TrainingConfig: dflash2_selector_loss_alpha: float = 1.0 # DSpark-specific parameters (used by DSpark trainer only) + # Precision of the frozen Draft LM head during training (SM90 FP8, CP1). + dspark_lm_head_precision: str = "bf16" dspark_num_anchors: int = 512 dspark_num_target_layers: int = 5 dspark_loss_decay_gamma: float = 4.0 diff --git a/aspec/inference/dspark_fp8_head.py b/aspec/inference/dspark_fp8_head.py new file mode 100644 index 0000000..56c2a38 --- /dev/null +++ b/aspec/inference/dspark_fp8_head.py @@ -0,0 +1,182 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Experimental SM90 Draft-only FP8 LM head for native SGLang DSpark decode. + +Reuses DSpark's native training-time rowwise FP8 projection and SGLang's native quant_method +dispatch. The Target module and its canonical BF16 parameter are never changed. +This is an explicit per-process Qwen3/TP1 benchmark adapter, not a serving flag. +No SGLang implementation is copied; source hashes bind the adapter to the tested +interfaces before installing it. Runtime FP8 caches are not checkpoint weights. +""" +from __future__ import annotations + +import functools +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import torch +from torch import nn + +from aspec.models.ops.dspark_fp8_lm_head import _project, quantize_activation + + +def _version(weight): + try: + return weight._version + except RuntimeError as error: + if "Inference tensors do not track version counter" not in str(error): + raise + return None + + +class DraftRowwiseFP8Method: + """Inference-only E4M3 GEMM; forward weight scale axis matches training B.""" + + @torch.no_grad() + def __init__(self, weight): + if weight.ndim != 2 or not weight.is_cuda or weight.dtype != torch.bfloat16: + raise ValueError("Draft FP8 inference requires a CUDA BF16 [vocab, hidden] weight") + if torch.cuda.get_device_capability(weight.device) != (9, 0): + raise ValueError("This inference prototype requires SM90") + if min(weight.shape) == 0 or any(size % 16 for size in weight.shape): + raise ValueError("Head dimensions must be positive multiples of 16") + if not bool(torch.isfinite(weight).all()): + raise ValueError("Head weight contains nonfinite values") + self.weight = weight + self.weight_version = _version(weight) + self.weight_pointer = weight.data_ptr() + bits, scales = quantize_activation(weight.detach()) + self.weight_t = bits.t() + self.scale_t = scales.t() + self.capture_shapes = set() + self.eager_shapes = set() + + @property + def cache_bytes(self): + return sum(t.numel() * t.element_size() for t in (self.weight_t, self.scale_t)) + + def apply(self, layer, hidden, bias=None): + if bias is not None or layer.weight is not self.weight: + raise ValueError("FP8 head view received a different weight or bias") + if self.weight.dtype != torch.bfloat16 or self.weight.data_ptr() != self.weight_pointer: + raise ValueError("Canonical Target head changed") + if self.weight_version is not None and _version(self.weight) != self.weight_version: + raise ValueError("Canonical head changed; rebuild before graph capture") + if hidden.ndim < 2 or hidden.shape[-1] != self.weight.shape[1]: + raise ValueError("Draft hidden dimension does not match the head") + if hidden.dtype != torch.bfloat16 or hidden.device != self.weight.device: + raise ValueError("Draft hidden must remain BF16 on the head device") + if torch.is_grad_enabled() and (hidden.requires_grad or self.weight.requires_grad): + raise ValueError("This projection is for inference, not training") + matrix = hidden.reshape(-1, hidden.shape[-1]) + shapes = self.capture_shapes if torch.cuda.is_current_stream_capturing() else self.eager_shapes + shapes.add(tuple(matrix.shape)) + if matrix.shape[0] == 0: + return hidden.new_empty((*hidden.shape[:-1], self.weight.shape[0])) + return _project(matrix, self.weight_t, self.scale_t).view( + *hidden.shape[:-1], self.weight.shape[0] + ) + + +class DraftFP8HeadView(nn.Module): + """Separate quant_method owner with a read-only alias of the Target weight.""" + + def __init__(self, target_head): + super().__init__() + self.weight = target_head.weight + self.org_vocab_size = int(target_head.org_vocab_size) + if self.weight.shape[0] < self.org_vocab_size: + raise ValueError("This experiment supports TP1 only") + self.quant_method = DraftRowwiseFP8Method(self.weight) + + +def install_sglang_draft_head(*, precision, evidence_dir, source_hashes): + """Install in every scheduler process before shared-head attachment/capture.""" + if precision not in {"bf16", "fp8"}: + raise ValueError("Unknown Draft head inference precision") + if importlib.metadata.version("sglang") != "0.5.21": + raise ValueError("The inference adapter is validated for SGLang 0.5.21") + from sglang.srt.layers import logits_processor + from sglang.srt.models import dspark + from sglang.srt.speculative.dspark_components import dspark_worker_v2 + + modules = {"dspark.py": dspark, "logits_processor.py": logits_processor, + "dspark_worker_v2.py": dspark_worker_v2} + actual = {name: hashlib.sha256(Path(module.__file__).read_bytes()).hexdigest() + for name, module in modules.items()} + if actual != source_hashes: + raise ValueError("SGLang serving sources differ from the reviewed binding") + if getattr(dspark, "_aspec_draft_head_precision", None) is not None: + raise ValueError("Draft head adapter is already installed in this process") + dspark._aspec_draft_head_precision = precision + destination = Path(evidence_dir) + original_attach = dspark.DSparkDraftMixin.attach_shared_modules + original_graph = dspark_worker_v2.DSparkWorkerV2.init_cuda_graphs + + def write(stage, data): + destination.mkdir(parents=True, exist_ok=True) + (destination / f"{stage}-{os.getpid()}.json").write_text( + json.dumps(data, indent=2) + "\n") + + @functools.wraps(original_attach) + def attach(self, *, embed_tokens, lm_head): + if type(self).__name__ != "Qwen3DSparkModel": + raise ValueError("Draft-only FP8 adapter is scoped to dense Qwen3 DSpark") + if logits_processor.should_apply_lm_head_quant_method(lm_head, lm_head.quant_method): + raise ValueError("Target head must use its native unquantized BF16 path") + original_attach(self, embed_tokens=embed_tokens, lm_head=lm_head) + if any(p.dtype != torch.bfloat16 for p in self.parameters() if p.is_floating_point()): + raise ValueError("Draft parameters must remain BF16") + before = (lm_head.weight.data_ptr(), _version(lm_head.weight), lm_head.quant_method) + if precision == "fp8": + self.lm_head = DraftFP8HeadView(lm_head) + if before != (lm_head.weight.data_ptr(), _version(lm_head.weight), lm_head.quant_method): + raise ValueError("Adapter modified the Target head") + data = { + "precision": precision, "model_class": type(self).__name__, + "canonical_head_dtype": str(lm_head.weight.dtype), + "draft_parameter_dtype": "torch.bfloat16", + "shared_weight_pointer": lm_head.weight.data_ptr(), + "target_quant_method": type(lm_head.quant_method).__name__, + "target_head_unchanged": True, + "draft_head_is_target_head": self.lm_head is lm_head, + "cache_bytes": self.lm_head.quant_method.cache_bytes if precision == "fp8" else 0, + "source_hashes": actual, + "kernel": "torch._scaled_mm E4M3 rowwise power-of-two, fast_accum=False" if precision == "fp8" else "native BF16", + "cuda_visible_devices": os.getenv("CUDA_VISIBLE_DEVICES"), + } + self._aspec_head_audit = SimpleNamespace(target_head=lm_head, before=before, data=data) + write("attached", data) + + @functools.wraps(original_graph) + def graph(self, *args, **kwargs): + result = original_graph(self, *args, **kwargs) + audit = getattr(self.draft_model, "_aspec_head_audit", None) + if audit is None: + raise ValueError("Draft head adapter was not activated before graph capture") + target_head = self.target_worker.model_runner.model.lm_head + if target_head is not audit.target_head: + raise ValueError("Target no longer owns the original BF16 head module") + if audit.before != (target_head.weight.data_ptr(), _version(target_head.weight), target_head.quant_method): + raise ValueError("Target head changed during graph capture") + sampler = self._draft_sampler + data = {**audit.data, + "target_parameter_dtypes": sorted({str(p.dtype) for p in + self.target_worker.model_runner.model.parameters() if p.is_floating_point()}), + "draft_graph_available": self.draft_model_runner.decode_cuda_graph_runner is not None, + "folded_proposal": sampler is not None, + "folded_sampling": sampler.folded_sampling if sampler is not None else None} + if precision == "fp8": + method = self.draft_model.lm_head.quant_method + data.update(fp8_cache_dtype=str(method.weight_t.dtype), + capture_shapes=sorted(method.capture_shapes), + eager_shapes=sorted(method.eager_shapes)) + write("graph-ready", data) + return result + + dspark.DSparkDraftMixin.attach_shared_modules = attach + dspark_worker_v2.DSparkWorkerV2.init_cuda_graphs = graph diff --git a/aspec/models/dspark.py b/aspec/models/dspark.py index 678c357..c96c166 100644 --- a/aspec/models/dspark.py +++ b/aspec/models/dspark.py @@ -105,6 +105,7 @@ def __init__( anchor_seed=anchor_seed, anchor_sampling_scope=anchor_sampling_scope, ) + self.fp8_lm_head = None # Runtime cache bound after the frozen Target weights are loaded. self.ce_loss_alpha = float(ce_loss_alpha) self.l1_loss_alpha = float(l1_loss_alpha) self.confidence_head_alpha = float(confidence_head_alpha) @@ -463,9 +464,7 @@ def forward( ).view(*teacher.shape[:-1], lm_head_weight.shape[0]) self.loss_head_cached_teacher_blocks_observed += counts[chunk] hidden = values[0] - cached_draft = F.linear( - hidden.reshape(-1, hidden.shape[-1]), lm_head_weight - ).view(*hidden.shape[:-1], lm_head_weight.shape[0]) + cached_draft = self._project_draft(hidden, lm_head_weight) self.loss_head_cached_draft_blocks_observed += counts[chunk] # One cache per original chunk and forward, never per model/step. # Keep only small FP32 L1 statistics across checkpoint replay; full @@ -550,6 +549,16 @@ def forward( (loss_numerator, local_den.detach()), ) + def _project_draft(self, hidden, head_weight): + """Project Draft states; Target projections retain their own BF16 path.""" + if self.training and self.fp8_lm_head is not None: + if head_weight is not self.fp8_lm_head.weight: + raise ValueError("FP8 LM head cache is bound to a different frozen weight") + return self.fp8_lm_head(hidden) + return F.linear(hidden.reshape(-1, hidden.shape[-1]), head_weight).view( + *hidden.shape[:-1], head_weight.shape[0] + ) + def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, aligned_target_hidden, eval_mask, weights, cached_target_logits=None, cached_draft_logits=None, cached_target_positions=None, loss_cache=None): @@ -557,11 +566,8 @@ def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, # Explicit token-major GEMM: a block slice can be non-contiguous even # though the complete hidden stream is contiguous. Do not broadcast a # vocabulary projection over many tiny [block_size, hidden] matrices. - logits = cached_draft_logits if cached_draft_logits is not None else F.linear( - hidden.reshape(-1, hidden.shape[-1]), head_weight - ).view( - *hidden.shape[:-1], head_weight.shape[0] - ) + logits = (cached_draft_logits if cached_draft_logits is not None + else self._project_draft(hidden, head_weight)) if self.draft_model.markov_head is not None: logits = self.draft_model.markov_head.apply_block_logits(logits, token_ids=prev_ids) vocab = logits.shape[-1] diff --git a/aspec/models/ops/dspark_fp8_lm_head.py b/aspec/models/ops/dspark_fp8_lm_head.py new file mode 100644 index 0000000..53cdd77 --- /dev/null +++ b/aspec/models/ops/dspark_fp8_lm_head.py @@ -0,0 +1,191 @@ +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Experimental frozen-head FP8 forward AND input-gradient projection on SM90. + +Independent implementation of row-scaled GEMM and a straight-through backward. +Inspired by SGLang PR 39622 (d3cacb55, cached frozen-head projection) and +torchao 4ab9d61d (different rowwise weight axes for output and input gradient). +No upstream implementation is copied. Uses PyTorch's existing scaled-mm kernel. + +DSpark constructs both disposable FP8 caches from the frozen Target weight +after initialization or checkpoint restore. Exports retain canonical BF16 weights. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl +from torch.autograd.function import once_differentiable + + +def quantize_rows(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """E4M3 with a power-of-two dequantization scale per GEMM output row. + + Scaling is reduced in FP32. An absolute floor bounds tiny/zero-row scales. + Quantization rounding is not differentiated; the custom backward implements + the selected approximate linear gradient, as in float8 training recipes. + """ + values = x.float() + scale = torch.exp2(torch.ceil(torch.log2(values.abs().amax(-1, keepdim=True).clamp_min(1e-12) / 448.0))) + bits = (values / scale).clamp(-448.0, 448.0).to(torch.float8_e4m3fn) + return bits, scale + + +@triton.jit +def _small_rows(X, Q, S, ROWS: tl.constexpr, WIDTH: tl.constexpr, BLOCK: tl.constexpr): + row = tl.program_id(0).to(tl.int64) + col = tl.arange(0, BLOCK) + value = tl.load(X + row * WIDTH + col, (row < ROWS) & (col < WIDTH), other=0).to(tl.float32) + maximum = tl.maximum(tl.max(tl.abs(value), 0), 1e-12) + scale = tl.exp2(tl.ceil(tl.log2(maximum / 448.0))) + quantized = tl.minimum(tl.maximum(value / scale, -448.0), 448.0) + tl.store(Q + row * WIDTH + col, quantized, col < WIDTH) + tl.store(S + row, scale) + + +@triton.jit +def _row_partials(X, P, WIDTH: tl.constexpr, TILES: tl.constexpr, BLOCK: tl.constexpr): + row = tl.program_id(0).to(tl.int64) + tile = tl.program_id(1) + col = tile * BLOCK + tl.arange(0, BLOCK) + value = tl.load(X + row * WIDTH + col, col < WIDTH, other=0).to(tl.float32) + maximum = tl.max(tl.abs(value), 0) + tl.store(P + row * TILES + tile, maximum) + + +@triton.jit +def _quantize_tiles(X, P, Q, S, ROWS: tl.constexpr, WIDTH: tl.constexpr, + TILES: tl.constexpr, REDUCE: tl.constexpr, BLOCK: tl.constexpr): + row = tl.program_id(0).to(tl.int64) + tile = tl.program_id(1) + indices = tl.arange(0, REDUCE) + partial = tl.load(P + row * TILES + indices, (row < ROWS) & (indices < TILES), other=0) + maximum = tl.maximum(tl.max(partial, 0), 1e-12) + scale = tl.exp2(tl.ceil(tl.log2(maximum / 448.0))) + col = tile * BLOCK + tl.arange(0, BLOCK) + value = tl.load(X + row * WIDTH + col, (row < ROWS) & (col < WIDTH), other=0).to(tl.float32) + quantized = tl.minimum(tl.maximum(value / scale, -448.0), 448.0) + tl.store(Q + row * WIDTH + col, quantized, col < WIDTH) + if tile == 0: + tl.store(S + row, scale) + + +def quantize_activation(x): + """Fuse zero padding with quantization; tile long vocabulary reductions. + + The first compiled-PyTorch prototype spent ~3 ms reducing a 151936-wide + gradient and materialized a padded BF16 copy. This two-pass reduction uses + bounded per-program state and emits the padded FP8 buffer directly. + """ + x = x.contiguous() + rows, width = x.shape + padded = triton.cdiv(rows, 16) * 16 + bits = torch.empty((padded, width), dtype=torch.float8_e4m3fn, device=x.device) + scale = torch.empty((padded, 1), dtype=torch.float32, device=x.device) + if width <= 8192: + _small_rows[(padded,)](x, bits, scale, rows, width, triton.next_power_of_2(width)) + else: + block = 4096 + tiles = triton.cdiv(width, block) + partials = torch.empty((rows, tiles), dtype=torch.float32, device=x.device) + _row_partials[(rows, tiles)](x, partials, width, tiles, block) + _quantize_tiles[(padded, tiles)]( + x, partials, bits, scale, rows, width, tiles, triton.next_power_of_2(tiles), block, + ) + return bits, scale + + +def _project(x, weight_t, scale_t): + rows = x.shape[0] + bits, scale = quantize_activation(x) + output = torch._scaled_mm( + bits, weight_t, scale_a=scale, scale_b=scale_t, + out_dtype=torch.bfloat16, use_fast_accum=False, + ) + return output[:rows] + + +class _FrozenHead(torch.autograd.Function): + @staticmethod + def forward(ctx, hidden, forward_weight_t, forward_scale_t, backward_weight_t, backward_scale_t): + ctx.input_shape = hidden.shape + ctx.save_for_backward(backward_weight_t, backward_scale_t) + matrix = hidden.reshape(-1, hidden.shape[-1]) + if matrix.shape[0] == 0: + return hidden.new_empty((*hidden.shape[:-1], forward_weight_t.shape[1])) + return _project(matrix, forward_weight_t, forward_scale_t).view( + *hidden.shape[:-1], forward_weight_t.shape[1] + ) + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + weight_t, scale_t = ctx.saved_tensors + if grad_output.numel() == 0: + grad_hidden = grad_output.new_empty(ctx.input_shape) + else: + matrix = grad_output.reshape(-1, grad_output.shape[-1]) + grad_hidden = _project(matrix, weight_t, scale_t).view(ctx.input_shape) + return grad_hidden, None, None, None, None + + +class FrozenFP8Head: + """Explicitly initialized, version-checked cache for one frozen BF16 head. + + Shape: canonical weight [vocabulary, hidden]. Both dimensions must divide 16. + The caller must rebuild after replacing/mutating the canonical weight. Cache + construction, validation and allocated bytes are visible to the benchmark. + """ + + def __init__(self, weight: torch.Tensor): + if weight.ndim != 2 or not weight.is_cuda or weight.dtype != torch.bfloat16: + raise ValueError("FP8 head requires a CUDA BF16 [vocabulary, hidden] weight") + if weight.requires_grad: + raise ValueError("FP8 head requires frozen weights") + if min(weight.shape) == 0 or any(size % 16 for size in weight.shape): + raise ValueError("FP8 head weight dimensions must be positive multiples of 16") + if torch.cuda.get_device_capability(weight.device) != (9, 0): + raise ValueError("This prototype is scoped to SM90") + if not bool(torch.isfinite(weight).all()): + raise ValueError("FP8 head weight contains nonfinite values") + self.weight = weight + self.version = weight._version + with torch.no_grad(): + forward_bits, forward_scale = quantize_rows(weight.contiguous()) + backward_bits, backward_scale = quantize_rows(weight.t().contiguous()) + self.forward_weight_t = forward_bits.t() + self.forward_scale_t = forward_scale.t() + self.backward_weight_t = backward_bits.t() + self.backward_scale_t = backward_scale.t() + + @property + def cache_bytes(self): + return sum(t.numel() * t.element_size() for t in ( + self.forward_weight_t, self.forward_scale_t, + self.backward_weight_t, self.backward_scale_t, + )) + + def __call__(self, hidden: torch.Tensor): + if self.weight._version != self.version or self.weight.requires_grad: + raise ValueError("Frozen head weight changed; rebuild the FP8 cache") + if hidden.ndim < 2 or hidden.shape[-1] != self.weight.shape[1]: + raise ValueError("Hidden input must end in the head hidden dimension") + if hidden.device != self.weight.device or hidden.dtype != torch.bfloat16: + raise ValueError("Hidden input must use the same CUDA device and BF16 dtype") + return _FrozenHead.apply( + hidden, self.forward_weight_t, self.forward_scale_t, + self.backward_weight_t, self.backward_scale_t, + ) diff --git a/aspec/planning.py b/aspec/planning.py index 29542fc..7fe410a 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -277,6 +277,17 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: f"Draft architecture {architecture!r} is outside the current DSpark/DFlash/DFlash2/EAGLE3 scope" ) + head_precision = _select(config, "training.dspark_lm_head_precision", "bf16") + if head_precision not in {"bf16", "fp8"}: + raise PlanCompileError("training.dspark_lm_head_precision must be 'bf16' or 'fp8'") + if head_precision == "fp8": + if family != "dspark" or _select(config, "training.draft_context_parallel_size", 1) != 1: + raise PlanCompileError("FP8 LM head training requires DSpark with CP1") + if _select(config, "training.compile_model", False): + raise PlanCompileError("FP8 LM head training requires compile_model=false") + if any(raw.get(f"{kind}_training_stage") == "indexer_warmup" for kind in ("csa", "qsa", "dsa")): + raise PlanCompileError("Indexer warmup does not run the FP8 LM head") + if family == "mtp": from aspec.models.draft.mtp import NativeMTPConfig @@ -1325,6 +1336,7 @@ def _workload( "loss_decay_gamma": _select(config, "training.dspark_loss_decay_gamma"), "ce_loss_alpha": _select(config, "training.dspark_ce_loss_alpha"), "l1_loss_alpha": _select(config, "training.dspark_l1_loss_alpha"), + "lm_head_precision": _select(config, "training.dspark_lm_head_precision", "bf16"), "confidence_head_alpha": _select( config, "training.dspark_confidence_head_alpha" ), diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index baea2b4..eb220ee 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -79,6 +79,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: fields = ("num_anchors", "num_target_layers", "loss_decay_gamma", "ce_loss_alpha", "l1_loss_alpha") if prefix == "dspark": + topology["training_objective"]["lm_head_precision"] = getattr(args, "dspark_lm_head_precision", "bf16") fields += ("confidence_head_alpha",) for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): @@ -146,6 +147,8 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec current_topology = _logical_training_topology(args, current_world_size) if getattr(args, "draft_algorithm", None) == "dspark": saved_objective = (saved_topology or {}).get("training_objective") or {} + if saved_objective.get("lm_head_precision", "bf16") != getattr(args, "dspark_lm_head_precision", "bf16"): + raise RuntimeError("Exact resume cannot change dspark_lm_head_precision; use weights-only initialization") for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): key = f"l1_{side}_{name}" diff --git a/aspec/training/dspark_trainer.py b/aspec/training/dspark_trainer.py index 0886983..f53586e 100644 --- a/aspec/training/dspark_trainer.py +++ b/aspec/training/dspark_trainer.py @@ -67,6 +67,20 @@ def __init__(self, args: Namespace): # Build hooks (override DFlashTrainer's defaults) # ------------------------------------------------------------------ + def _init_target_lm_head(self, target_model_path): + super()._init_target_lm_head(target_model_path) + if getattr(self.args, "dspark_lm_head_precision", "bf16") == "fp8": + from aspec.models.ops.dspark_fp8_lm_head import FrozenFP8Head + + self.dflash.fp8_lm_head = FrozenFP8Head(self.target_lm_head.lm_head.weight) + + def get_feature_contract_status(self): + status = super().get_feature_contract_status() + cache = getattr(getattr(self, "dflash", None), "fp8_lm_head", None) + if cache is not None: + status["dspark_lm_head"] = {"training_precision": "fp8", "cache_bytes": cache.cache_bytes} + return status + def _build_draft_model(self, config): if isinstance(config, K3DSparkConfig): return K3DSparkModel(config) diff --git a/configs/recipes/qwen3_8b_dspark.yaml b/configs/recipes/qwen3_8b_dspark.yaml index 879eeaa..1187276 100644 --- a/configs/recipes/qwen3_8b_dspark.yaml +++ b/configs/recipes/qwen3_8b_dspark.yaml @@ -47,6 +47,7 @@ training: warmup_ratio: 0.04 dflash_block_size: 7 dflash_packing: true + dspark_lm_head_precision: bf16 # Use fp8 on SM90 for Draft forward and dX. dspark_num_anchors: 512 dspark_num_target_layers: 5 dspark_loss_decay_gamma: 4.0 diff --git a/tests/cpu/test_planning.py b/tests/cpu/test_planning.py index 4e97c9c..63cc1eb 100644 --- a/tests/cpu/test_planning.py +++ b/tests/cpu/test_planning.py @@ -57,6 +57,42 @@ def _local_qwen_target(tmp_path, *, max_position_embeddings=262144): return target +@pytest.mark.parametrize("precision", ["bf16", "fp8"]) +def test_dspark_head_precision_reaches_trainer_and_resume(precision): + from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, + ) + + plan = compile_execution_plan(_config(**{"training.dspark_lm_head_precision": precision})) + args = plan.to_runtime_args() + assert args.dspark_lm_head_precision == plan.workload.objective["lm_head_precision"] == precision + topology = _logical_training_topology(args, args.world_size) + metadata = {"world_size": args.world_size, "training_topology": topology} + assert topology["training_objective"]["lm_head_precision"] == precision + _validate_exact_resume_topology(args, metadata) + args.dspark_lm_head_precision = "bf16" if precision == "fp8" else "fp8" + with pytest.raises(RuntimeError, match="dspark_lm_head_precision"): + _validate_exact_resume_topology(args, metadata) + del topology["training_objective"]["lm_head_precision"] + args.dspark_lm_head_precision = "bf16" + _validate_exact_resume_topology(args, metadata) + args.dspark_lm_head_precision = "fp8" + with pytest.raises(RuntimeError, match="dspark_lm_head_precision"): + _validate_exact_resume_topology(args, metadata) + + +@pytest.mark.parametrize("options", [ + {"training.dspark_lm_head_precision": "int8"}, + {"training.dspark_lm_head_precision": "fp8", "training.draft_context_parallel_size": 2}, + {"training.dspark_lm_head_precision": "fp8", "training.compile_model": True}, + {"training.dspark_lm_head_precision": "fp8", "model.draft_model_config": str(_DRAFT_CONFIG.with_name("dflash_draft_config.json"))}, +]) +def test_invalid_dspark_head_precision_is_rejected(options): + with pytest.raises(PlanCompileError, match="LM head|lm_head_precision"): + compile_execution_plan(_config(**options)) + + def test_reference_config_compiles_deterministically(): config = _config() diff --git a/tests/test_fp8_dspark_inference.py b/tests/test_fp8_dspark_inference.py new file mode 100644 index 0000000..de3b9e3 --- /dev/null +++ b/tests/test_fp8_dspark_inference.py @@ -0,0 +1,118 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Draft FP8 dispatch must preserve the shared Target and CUDA graph results.""" +import hashlib +import os +from pathlib import Path + +import pytest +import torch + +from aspec.inference.dspark_fp8_head import DraftFP8HeadView, install_sglang_draft_head +from aspec.models.ops.dspark_fp8_lm_head import quantize_rows + +pytestmark = pytest.mark.skipif( + os.environ.get("ASPEC_RUN_GPU_INTEGRATION") != "1" or not torch.cuda.is_available() + or torch.cuda.get_device_capability() != (9, 0), + reason="requires an explicit SM90 inference check", +) + + +def head(): + value = torch.nn.Linear(128, 256, bias=False).to("cuda", dtype=torch.bfloat16) + value.org_vocab_size = 256 + value.quant_method = None + return value + + +@pytest.mark.parametrize("rows", [0, 1, 7, 14, 28, 56]) +@torch.no_grad() +def test_native_dispatch_changes_only_the_draft_view(rows): + from sglang.srt.models.dspark import project_through_lm_head + + torch.manual_seed(148) + target = head() + canonical = target.weight.clone() + hidden = torch.randn(rows, 128, device="cuda", dtype=torch.bfloat16) + target_logits = project_through_lm_head(hidden, target) + draft = DraftFP8HeadView(target) + actual = project_through_lm_head(hidden, draft) + bits, scale = quantize_rows(canonical) + method = draft.quant_method + # The serving weight recipe must be identical to the training forward cache. + assert torch.equal(method.weight_t.t().view(torch.uint8), bits.view(torch.uint8)) + assert torch.equal(method.scale_t.t(), scale) + if rows: + x, xs = quantize_rows(hidden) + expected = (x.float() * xs) @ (bits.float() * scale).t() + torch.testing.assert_close(actual, expected.bfloat16(), atol=1e-3, rtol=1e-2) + assert actual.shape == (rows, 256) and actual.dtype == torch.bfloat16 + assert draft.weight is target.weight + assert target.quant_method is None + assert torch.equal(target.weight, canonical) + assert torch.equal(project_through_lm_head(hidden, target), target_logits) + assert method.cache_bytes == 256 * 128 + 256 * 4 + + +@torch.no_grad() +def test_native_fp8_head_replays_in_cuda_graph_and_rejects_stale_weights(): + from sglang.srt.models.dspark import project_through_lm_head + + target = head() + draft = DraftFP8HeadView(target) + hidden = torch.randn(7, 128, device="cuda", dtype=torch.bfloat16) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + project_through_lm_head(hidden, draft) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = project_through_lm_head(hidden, draft) + for _ in range(3): + hidden.copy_(torch.randn_like(hidden)) + graph.replay() + expected = project_through_lm_head(hidden, draft) + assert torch.equal(captured, expected) + assert (7, 128) in draft.quant_method.capture_shapes + target.weight.add_(1) + with pytest.raises(ValueError, match="Canonical head changed"): + project_through_lm_head(hidden, draft) + + +@torch.no_grad() +def test_unsupported_head_dtype_and_partition_fail_closed(): + target = head() + target.org_vocab_size = 512 + with pytest.raises(ValueError, match="TP1"): + DraftFP8HeadView(target) + target.org_vocab_size = 256 + target.to(dtype=torch.float32) + with pytest.raises(ValueError, match="BF16"): + DraftFP8HeadView(target) + + +def test_process_installer_keeps_native_target_projection(monkeypatch, tmp_path): + from sglang.srt.layers import logits_processor + from sglang.srt.models import dspark + from sglang.srt.speculative.dspark_components import dspark_worker_v2 + + modules = {"dspark.py": dspark, "logits_processor.py": logits_processor, + "dspark_worker_v2.py": dspark_worker_v2} + binding = {name: hashlib.sha256(Path(module.__file__).read_bytes()).hexdigest() + for name, module in modules.items()} + projection = dspark.project_through_lm_head + dispatch = logits_processor.should_apply_lm_head_quant_method + attach = dspark.DSparkDraftMixin.attach_shared_modules + graph = dspark_worker_v2.DSparkWorkerV2.init_cuda_graphs + monkeypatch.setattr(dspark, "_aspec_draft_head_precision", None, raising=False) + monkeypatch.setattr(dspark.DSparkDraftMixin, "attach_shared_modules", attach) + monkeypatch.setattr(dspark_worker_v2.DSparkWorkerV2, "init_cuda_graphs", graph) + with pytest.raises(ValueError, match="sources differ"): + install_sglang_draft_head(precision="fp8", evidence_dir=tmp_path, source_hashes={}) + install_sglang_draft_head(precision="fp8", evidence_dir=tmp_path, source_hashes=binding) + assert dspark.project_through_lm_head is projection + assert logits_processor.should_apply_lm_head_quant_method is dispatch + assert dspark.DSparkDraftMixin.attach_shared_modules is not attach + assert dspark_worker_v2.DSparkWorkerV2.init_cuda_graphs is not graph diff --git a/tests/test_fp8_lm_head.py b/tests/test_fp8_lm_head.py new file mode 100644 index 0000000..11a68d1 --- /dev/null +++ b/tests/test_fp8_lm_head.py @@ -0,0 +1,182 @@ +# Copyright 2026 Ant Group Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 + +"""SM90 checks for the standalone FP8 head experiment; explicit GPU opt-in.""" + +import os + +import pytest +import torch +import torch.nn.functional as F + +from aspec.models.ops.dspark_fp8_lm_head import FrozenFP8Head, quantize_activation + +pytestmark = pytest.mark.skipif( + os.environ.get("ASPEC_RUN_GPU_INTEGRATION") != "1" or not torch.cuda.is_available() + or torch.cuda.get_device_capability() != (9, 0), + reason="requires ASPEC_RUN_GPU_INTEGRATION=1 and SM90 CUDA", +) + + +def reference_rows(x): + # Independent FP64 scaling arithmetic, with explicit dequantization. + source = x.double() + amax = source.abs().max(dim=1, keepdim=True).values.clamp_min(1e-12) + # ldexp constructs exact powers of two; generic CUDA pow can add FP64 + # rounding noise even when the mathematical result is exactly binary. + exponent = torch.ceil(torch.log2(amax / 448.0)).to(torch.int32) + scale = torch.ldexp(torch.ones_like(amax), exponent) + bits = (source / scale).clamp(-448, 448).to(torch.float8_e4m3fn) + return bits.double() * scale + + +@pytest.mark.parametrize("rows", [0, 1, 15, 16, 17, 63]) +@pytest.mark.parametrize("strided", [False, True]) +def test_both_gemms_match_independent_quantized_reference(rows, strided): + torch.manual_seed(901) + weight = torch.randn(48, 32, device="cuda", dtype=torch.bfloat16) * 0.1 + source = torch.randn(rows, 64 if strided else 32, device="cuda", dtype=torch.bfloat16) + hidden = (source[:, ::2] if strided else source).detach().requires_grad_() + head = FrozenFP8Head(weight) + output = head(hidden) + grad = torch.randn(rows, 48, device="cuda", dtype=torch.bfloat16) * 0.001 + output.backward(grad) + if rows: + expected = reference_rows(hidden.detach()) @ reference_rows(weight).t() + expected_grad = reference_rows(grad) @ reference_rows(weight.t()).t() + torch.testing.assert_close(output, expected.bfloat16(), atol=1e-3, rtol=1e-2) + torch.testing.assert_close(hidden.grad, expected_grad.bfloat16(), atol=1e-5, rtol=1e-2) + else: + assert output.shape == (0, 48) + assert hidden.grad.shape == hidden.shape + assert weight.grad is None + + +def test_zero_rows_and_stale_weight_cache(): + weight = torch.zeros(48, 32, device="cuda", dtype=torch.bfloat16) + head = FrozenFP8Head(weight) + hidden = torch.zeros(2, 17, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + head(hidden).sum().backward() + assert torch.count_nonzero(hidden.grad) == 0 + assert torch.isfinite(head(hidden)).all() + weight.add_(1) + with pytest.raises(ValueError, match="changed"): + head(hidden) + + +def test_trainable_head_is_rejected(): + weight = torch.zeros(48, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + with pytest.raises(ValueError, match="frozen"): + FrozenFP8Head(weight) + + +def test_gradient_quality_against_bf16_and_checkpoint_recomputation(): + from torch.utils.checkpoint import checkpoint + + torch.manual_seed(73) + weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * 0.02 + hidden = torch.randn(33, 128, device="cuda", dtype=torch.bfloat16, requires_grad=True) + head = FrozenFP8Head(weight) + labels = torch.arange(33, device="cuda") % 256 + expected = torch.autograd.grad(F.cross_entropy(F.linear(hidden, weight).float(), labels), hidden)[0] + actual = torch.autograd.grad(F.cross_entropy(head(hidden).float(), labels), hidden)[0] + replay = torch.autograd.grad( + checkpoint(lambda x: F.cross_entropy(head(x).float(), labels), hidden, use_reentrant=False), hidden + )[0] + # A bounded random-fixture smoke, not a real-model quality threshold. + assert (actual.float() - expected.float()).norm() / expected.float().norm() < 0.08 + torch.testing.assert_close(actual, replay, rtol=0, atol=0) + + +def test_production_vocabulary_backward_reduction(): + # The backward reduction dimension is the entire vocabulary, much larger + # than the forward hidden width. Exercise the actual Qwen3-8B vocabulary. + torch.manual_seed(19) + weight = torch.randn(151936, 128, device="cuda", dtype=torch.bfloat16) * 0.02 + hidden = torch.randn(17, 128, device="cuda", dtype=torch.bfloat16, requires_grad=True) + upstream = torch.randn(17, 151936, device="cuda", dtype=torch.bfloat16) * 1e-5 + head = FrozenFP8Head(weight) + output = head(hidden) + actual, = torch.autograd.grad(output, hidden, upstream) + expected = upstream.float() @ weight.float() + assert torch.isfinite(actual).all() + assert (actual.float() - expected).norm() / expected.norm() < 0.08 + + +@pytest.mark.parametrize("width", [32, 4096, 151936]) +def test_tiled_quantization_and_padding_match_independent_reference(width): + torch.manual_seed(51) + values = torch.randn(17, width, device="cuda", dtype=torch.bfloat16) + values *= torch.logspace(-8, 2, 17, device="cuda")[:, None] + values[0].zero_() + values[3, -1] = 448 + bits, scale = quantize_activation(values) + actual = bits[:17].double() * scale[:17].double() + torch.testing.assert_close(actual, reference_rows(values), rtol=0, atol=0) + assert bits.shape == (32, width) + assert torch.count_nonzero(bits[17:].float()) == 0 + assert torch.isfinite(scale).all() + + +def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch): + from types import SimpleNamespace + + from aspec.models.draft.dspark import AcceptRatePredictor, VanillaMarkov + from aspec.models.dspark import DSparkModel + from aspec.training.dflash_trainer import DFlashTrainer + from aspec.training.dspark_trainer import DSparkTrainer + + torch.manual_seed(82) + draft = torch.nn.Module() + draft.markov_head = VanillaMarkov(vocab_size=256, markov_rank=16) + draft.confidence_head = AcceptRatePredictor(144) + draft.confidence_head_with_markov = True + model = DSparkModel(draft).cuda().to(torch.bfloat16) + hidden = torch.randn(1, 3, 7, 128, device="cuda", dtype=torch.bfloat16, requires_grad=True) + teacher = torch.randn_like(hidden, requires_grad=False) + weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * 0.02 + ids = torch.randint(0, 256, (1, 3, 7), device="cuda") + mask = torch.ones_like(ids, dtype=torch.float32) + weights = mask * torch.exp(-torch.arange(7, device="cuda").float() / 4) + arguments = (hidden, ids, ids.roll(1, -1), weight, teacher, mask, weights) + model.eval() + with torch.no_grad(): + baseline = model._prediction_terms(*arguments) + before = {k: v.clone() for k, v in model.state_dict().items()} + trainer = DSparkTrainer.__new__(DSparkTrainer) + trainer.args = SimpleNamespace(dspark_lm_head_precision="fp8") + trainer.dflash = model + trainer.target_lm_head = SimpleNamespace(lm_head=SimpleNamespace(weight=weight)) + monkeypatch.setattr(DFlashTrainer, "_init_target_lm_head", lambda self, path: None) + trainer._init_target_lm_head("test-model") + assert model.fp8_lm_head.cache_bytes > 0 + assert model.state_dict().keys() == before.keys() + for key, value in model.state_dict().items(): + torch.testing.assert_close(value, before[key], rtol=0, atol=0) + with torch.no_grad(): + validation = model._prediction_terms(*arguments) + for actual, expected in zip(validation, baseline, strict=True): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + native_linear = F.linear + teacher_inputs = [] + + def observe(source, projection, bias=None): + if projection is weight: + teacher_inputs.append(source.detach().clone()) + return native_linear(source, projection, bias) + + monkeypatch.setattr(F, "linear", observe) + model.train() + terms = model._prediction_terms(*arguments) + (0.1 * terms[0] + 0.9 * terms[1] + terms[2]).backward() + assert len(teacher_inputs) == 1 + torch.testing.assert_close(teacher_inputs[0], teacher.reshape(-1, 128), rtol=0, atol=0) + assert torch.isfinite(hidden.grad).all() and hidden.grad.norm() > 0 + assert weight.grad is None + assert all(p.dtype == torch.bfloat16 and p.grad is not None and torch.isfinite(p.grad).all() + for p in model.parameters()) + with pytest.raises(ValueError, match="different frozen weight"): + model._project_draft(hidden, weight.clone()) diff --git a/tools/performance/torchspec_benchmark.py b/tools/performance/torchspec_benchmark.py index 9a7ce01..9adc679 100644 --- a/tools/performance/torchspec_benchmark.py +++ b/tools/performance/torchspec_benchmark.py @@ -570,6 +570,8 @@ def run_native(args): compile_model=False, load_path=None, resume=None, checkpoint_dir=None, enable_perf_metrics=True, num_train_steps=args.warmup + args.steps, ) + if namespace == "aspec": + config["dspark_lm_head_precision"] = args.dspark_lm_head_precision args.result.parent.mkdir(parents=True, exist_ok=True) args.initial.parent.mkdir(parents=True, exist_ok=True) if world_size > 1: @@ -850,6 +852,7 @@ def main(): ) parser.add_argument("--features", type=Path) parser.add_argument("--framework", choices=("aspec", "torchspec", "angelspec")) + parser.add_argument("--dspark-lm-head-precision", choices=("bf16", "fp8"), default="bf16") parser.add_argument("--embedding-key", default="model.embed_tokens.weight") parser.add_argument("--attention-backend", choices=("cudnn_flex_attention", "fa4"), default="cudnn_flex_attention", help="ASpec backend; native baselines keep their own FlexAttention backend.") @@ -876,6 +879,8 @@ def main(): help="Fail unless every ASpec Draft layer observes this Q/K preparation backend.", ) args = parser.parse_args() + if args.dspark_lm_head_precision == "fp8" and (args.mode != "run" or args.framework != "aspec"): + parser.error("FP8 LM head requires run --framework aspec") required = { "data": ("source", "output"), "features": ("data", "output"), "run": ("features", "framework", "initial", "result"),