Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion aspec/config/draft_algorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

DRAFT_ALGORITHMS = {
"NativeMTPDraftModel": "mtp",
"MedusaDraftModel": "medusa",
"DFlashDraftModel": "dflash",
"DFlash2DraftModel": "dflash2",
"DSparkDraftModel": "dspark",
Expand Down Expand Up @@ -65,7 +66,8 @@ def draft_model_semantics(config) -> dict:
"vocab_size", "draft_vocab_size", "hidden_act", "rms_norm_eps",
"max_position_embeddings", "rope_theta", "rope_scaling", "rope_parameters",
"target_layer_ids", "num_target_layers", "target_num_hidden_layers",
"num_aux_hidden_states", "fc_norm", "norm_output", "mask_token_id",
"num_aux_hidden_states", "medusa_num_heads", "medusa_num_layers", "fc_norm",
"norm_output", "mask_token_id",
"lm_head_rank_divisor", "attention_mode", "q_lora_rank", "kv_lora_rank",
"qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim", "block_size",
"conv_kernel_size", "conv_group_size", "selector_rank", "selector_top_k",
Expand Down
9 changes: 9 additions & 0 deletions aspec/config/draft_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@
def resolve_draft_head_layout(raw: Mapping) -> dict:
"""Resolve the physical Q/KV head layout used by DFlash/DSpark attention."""

if raw.get("architectures") == ["MedusaDraftModel"]:
return {"type": "heads_only", "query_heads": None, "key_value_heads": None,
"query_heads_per_key_value_head": None, "head_dim": None,
"head_dim_source": "not_applicable"}

def positive_integer(name: str, default: int) -> int:
value = raw.get(name, default)
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
Expand Down Expand Up @@ -115,6 +120,10 @@ def positive_integer(name: str, default: int) -> int:


def resolve_draft_attention(raw: Mapping) -> dict:
if raw.get("architectures") == ["MedusaDraftModel"]:
return {"sliding_window": None, "layer_types": [], "compress_ratios": None,
"csa": None, "qsa": None, "context_windows": [],
"window_semantics": "none", "draft_block_attention": "parallel_heads"}
layers = raw.get("num_hidden_layers", 1)
if isinstance(layers, bool) or not isinstance(layers, int) or layers < 1:
raise ValueError("Draft num_hidden_layers must be a positive integer")
Expand Down
2 changes: 2 additions & 0 deletions aspec/config/draft_structure.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ def resolve_draft_structure(raw: Mapping) -> dict:
from aspec.config.eagle4 import eagle4_structure

return eagle4_structure(raw)
if raw.get("architectures") == ["MedusaDraftModel"]:
return {"ffn_type": "heads_only"}
ffn_type = raw.get("ffn_type", "dense")
if ffn_type not in {"dense", "qwen3_moe", "ling_moe"}:
raise ValueError(f"Unsupported Draft ffn_type={ffn_type!r}")
Expand Down
18 changes: 18 additions & 0 deletions aspec/config/medusa_qwen3_8b_draft_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"architectures": ["MedusaDraftModel"],
"model_type": "aspec_medusa",
"hidden_size": 4096,
"target_hidden_size": 4096,
"vocab_size": 151936,
"draft_vocab_size": 151936,
"target_num_hidden_layers": 36,
"target_layer_ids": [35],
"num_target_layers": 1,
"num_aux_hidden_states": 1,
"medusa_num_heads": 5,
"medusa_num_layers": 1,
"tie_word_embeddings": false,
"norm_output": false,
"fc_norm": false,
"torch_dtype": "bfloat16"
}
5 changes: 3 additions & 2 deletions aspec/feature_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ def build_target_feature_contract(
"layout": "sequence",
},
}
if drafter_family == "mtp":
if drafter_family in {"mtp", "medusa"}:
if len(aux_hidden_state_layers) != 1 or store_last_hidden_states:
raise FeatureContractError("Native MTP requests one post-norm feature, without a duplicate teacher tensor")
raise FeatureContractError(
"Native MTP and Medusa request one post-norm feature, without a duplicate teacher tensor")
tensors["hidden_states"] = {
"semantic": "target_final_hidden_state", "dtype": "bfloat16",
"shape": ["sequence", target_hidden_size], "layout": "sequence_hidden",
Expand Down
2 changes: 1 addition & 1 deletion aspec/inference/sglang_v0521.py
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,7 @@ def training_engine_kwargs(kwargs, args):
"max_sequence_length": args.max_seq_length,
"store_last_hidden_states": args.store_last_hidden_states,
"direct_prefill": args.target_prefill_memory_optimized,
"final_hidden_as_aux": getattr(args, "draft_algorithm", None) == "mtp",
"final_hidden_as_aux": getattr(args, "draft_algorithm", None) in {"mtp", "medusa"},
"attention_backend": args.attention_backend,
**({"mtp": args.aspec_feature_contract["mtp"], "target_model_path": args.target_model_path}
if (getattr(args, "aspec_feature_contract", None) or {}).get("mtp") is not None else {}),
Expand Down
3 changes: 3 additions & 0 deletions aspec/models/draft/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
)
from aspec.models.draft.eagle4 import Eagle4DraftModel
from aspec.models.draft.llama3_eagle import LlamaForCausalLMEagle3
from aspec.models.draft.medusa import MedusaConfig, MedusaDraftModel
from aspec.models.draft.mtp import NativeMTPConfig, NativeMTPDraftModel
from aspec.utils.logging import logger

Expand All @@ -46,6 +47,7 @@ class AutoEagle3DraftModel(AutoModelForCausalLMBase):
Eagle4Config: Eagle4DraftModel,
DeepseekV3Config: Eagle3DeepseekV2ForCausalLM,
NativeMTPConfig: NativeMTPDraftModel,
MedusaConfig: MedusaDraftModel,
DFlashConfig: DFlashDraftModel,
DFlash2Config: DFlash2DraftModel,
DSparkConfig: DSparkDraftModel,
Expand All @@ -68,6 +70,7 @@ class AutoDraftModelConfig:
"LlamaForCausalLMEagle4": Eagle4Config,
"Eagle3DeepseekV2ForCausalLM": DeepseekV3Config,
"NativeMTPDraftModel": NativeMTPConfig,
"MedusaDraftModel": MedusaConfig,
"DFlashDraftModel": DFlashConfig,
"DFlash2DraftModel": DFlash2Config,
"Qwen3DSparkModel": DSparkConfig,
Expand Down
11 changes: 10 additions & 1 deletion aspec/models/draft/medusa.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,12 @@ def forward(self, x):
class MedusaDraftModel(nn.Module):
config_class = MedusaConfig

def __init__(self, config):
def __init__(self, config, attention_backend="sdpa"):
super().__init__()
if attention_backend != "sdpa":
raise NotImplementedError("Medusa heads have no attention; use the sdpa backend")
self.config = config
self.attention_backend = attention_backend
self.medusa_num_heads = config.medusa_num_heads
self.medusa_num_layers = config.medusa_num_layers
self.hidden_size = config.hidden_size
Expand All @@ -114,3 +117,9 @@ def __init__(self, config):
def forward(self, hidden_states):
"""[B, S, H] frozen Target features -> [K, B, S, V] per-head logits."""
return torch.stack([head(hidden_states) for head in self.medusa_head], dim=0)

def load_embedding(self, target_model_path, embedding_key=None):
"""Heads-only: the draft reads Target features and owns no token embedding."""

def freeze_embedding(self):
"""Heads-only: the draft reads Target features and owns no token embedding."""
52 changes: 50 additions & 2 deletions aspec/planning.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,34 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]:
if _select(config, "inference.store_last_hidden_states"):
raise PlanCompileError("Native MTP publishes final post-norm features once; set store_last_hidden_states=false")

if family == "medusa":
from aspec.models.draft.medusa import MedusaConfig

try:
raw = MedusaConfig.from_dict(raw).to_dict()
except (ValueError, TypeError) as exc:
raise PlanCompileError(str(exc)) from exc
backend = _select(config, "training.attention_backend")
if backend != "sdpa":
raise PlanCompileError("Medusa heads have no attention; use training.attention_backend=sdpa")
if _select(config, "training.dflash_packing", False):
raise PlanCompileError("Medusa does not implement sequence packing yet")
if _select(config, "training.draft_context_parallel_size", 1) != 1:
raise PlanCompileError("Medusa currently supports draft_context_parallel_size=1")
if _select(config, "training.micro_batch_size", 1) != 1:
raise PlanCompileError("Medusa currently supports micro_batch_size=1")
if _select(config, "inference.store_last_hidden_states", False):
raise PlanCompileError(
"Medusa publishes final post-norm features once; set store_last_hidden_states=false")
if _select(config, "training.compile_model", False):
raise PlanCompileError("Medusa whole-model compile has not been enabled")
if _select(config, "model.freeze_lm_head", False):
raise PlanCompileError("Medusa heads own trainable vocabulary projections")
if _select(config, "training.eagle3_num_anchors", 0):
raise PlanCompileError("Medusa does not implement anchored training")
if _select(config, "training.ploss_weights") is not None:
raise PlanCompileError("Medusa fixes uniform per-head loss weights")

if family == "eagle4":
from aspec.config.eagle4 import Eagle4Config, eagle4_policy

Expand Down Expand Up @@ -618,6 +646,8 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]:
"target_hidden_size": raw.get("target_hidden_size"),
"target_num_hidden_layers": raw.get("target_num_hidden_layers"),
"target_layer_ids": target_layer_ids,
"medusa_num_heads": raw.get("medusa_num_heads"),
"medusa_num_layers": raw.get("medusa_num_layers"),
}


Expand Down Expand Up @@ -1214,7 +1244,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None:
f"got {declared_target_layers} and {expected_target_layers}"
)
training_target_layers = (
expected_target_layers if draft["family"] in {"eagle3", "eagle4", "mtp"} else getattr(
expected_target_layers if draft["family"] in {"eagle3", "eagle4", "mtp", "medusa"} else getattr(
args,
"dspark_num_target_layers" if draft["family"] == "dspark" else "dflash_num_target_layers",
)
Expand All @@ -1227,7 +1257,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None:

block_size = getattr(args, "dflash_block_size", 16)
min_loss_tokens = getattr(args, "min_loss_tokens", 0)
if draft["family"] not in {"eagle3", "eagle4", "mtp"} and min_loss_tokens < 2 * block_size:
if draft["family"] not in {"eagle3", "eagle4", "mtp", "medusa"} and min_loss_tokens < 2 * block_size:
raise PlanCompileError(
f"{draft['family']} requires dataset.min_loss_tokens >= "
f"2 * training.dflash_block_size ({min_loss_tokens} < {2 * block_size})"
Expand All @@ -1253,6 +1283,11 @@ def _runtime_values(config: DictConfig, draft: Mapping[str, Any], *, source_cont
args.ttt_length = args.mtp_steps # common recurrent trainer's metric slots
args.ploss_weights = mtp_step_weights(args.mtp_steps, args.mtp_step_weight_beta)
args.freeze_lm_head = True
elif args.draft_algorithm == "medusa":
args.ttt_length = draft["medusa_num_heads"]
# The reference sums per-head mean CEs; the trainer applies ttt weights.
args.ploss_weights = [1.0] * args.ttt_length
args.freeze_lm_head = False
args.source_context_parallel_size = (args.draft_context_parallel_size
if source_context_parallel_size is None else source_context_parallel_size)
if args.draft_algorithm in {"eagle3", "eagle4"} and args.ploss_weights is None:
Expand Down Expand Up @@ -1332,6 +1367,19 @@ def _workload(
for side in ("target", "draft"):
for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)):
objective[f"l1_{side}_{name}"] = _select(config, f"training.dspark_l1_{side}_{name}", default)
elif family == "medusa":
objective = {
"family": "medusa",
"loss": "parallel_future_token_ce",
"num_heads": draft["medusa_num_heads"],
"num_layers": draft["medusa_num_layers"],
"prediction_start": 2,
"head_weighting": "sum_of_per_head_mean_ce",
"frozen_shared": ["target_features"],
"verifier_feature_stage": "post_norm",
"vocab_scope": "full_target_vocabulary",
"gradient_checkpointing": _select(config, "training.gradient_checkpointing"),
}
else:
objective = {
"family": family,
Expand Down
9 changes: 8 additions & 1 deletion aspec/training/checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]:
"step_weights": args.ploss_weights,
"loss": "teacher_forced_native_mtp_ce", "frozen_shared": ["embedding", "lm_head"],
}
elif getattr(args, "draft_algorithm", None) == "medusa":
topology["training_objective"] = {
"ttt_length": args.ttt_length,
"ploss_weights": args.ploss_weights,
"loss": "parallel_future_token_ce",
"head_weighting": "sum_of_per_head_mean_ce",
}
elif getattr(args, "draft_algorithm", None) in {"eagle3", "eagle4"}:
topology["training_objective"] = {
key: getattr(args, key, None)
Expand Down Expand Up @@ -96,7 +103,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]:
if "training_objective" in topology:
fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps",
"max_grad_norm", "seed")
fields += (("freeze_lm_head",) if args.draft_algorithm in {"eagle3", "eagle4", "mtp"}
fields += (("freeze_lm_head",) if args.draft_algorithm in {"eagle3", "eagle4", "mtp", "medusa"}
else ("deterministic_anchor_sampling",))
for key in fields:
if hasattr(args, key):
Expand Down
44 changes: 44 additions & 0 deletions aspec/training/medusa_trainer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Copyright 2026 Ant Group Co., Ltd.
# SPDX-License-Identifier: Apache-2.0

"""Heads-only Medusa uses the existing optimizer/checkpoint/ownership training path."""

from types import SimpleNamespace

from aspec.models.medusa import MedusaModel
from aspec.training.eagle3_trainer import Eagle3Trainer
from aspec.training.trainer import Trainer


class MedusaTrainer(Eagle3Trainer):
def _create_training_model(self, draft_model):
return MedusaModel(
draft_model,
gradient_checkpointing=getattr(self.args, "gradient_checkpointing", False),
)

def _prepare_algorithm_training(self, draft_model):
if getattr(self.args, "freeze_lm_head", False):
raise ValueError("Medusa heads own trainable vocabulary projections")

def _init_target_lm_head(self, target_model_path):
# CE needs no teacher distribution. The trainer's unconditional
# weight/norm reads at init only need an object; reuse an existing head.
self.target_lm_head = SimpleNamespace(
lm_head=self.draft_model.medusa_head[0][-1], norm=None)

def get_feature_contract_status(self):
status = dict(Trainer.get_feature_contract_status(self))
status["medusa_num_heads"] = getattr(self.args, "ttt_length", None)
status["medusa_num_layers"] = getattr(
getattr(self, "draft_model", None), "medusa_num_layers", None)
return status

def _forward(self, batch):
return self.model(
input_ids=batch["input_ids"].cuda(),
hidden_states=batch["hidden_states"].cuda(),
loss_mask=batch["loss_mask"].squeeze(-1).cuda()
if batch["loss_mask"].ndim == 3 else batch["loss_mask"].cuda(),
attention_mask=batch["attention_mask"].cuda(),
)
6 changes: 5 additions & 1 deletion aspec/training/trainer_actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ def _trainer_class_for_config(draft_model_config):
from aspec.training.mtp_trainer import MTPTrainer

return MTPTrainer
if getattr(draft_model_config, "architectures", None) == ["MedusaDraftModel"]:
from aspec.training.medusa_trainer import MedusaTrainer

return MedusaTrainer
if isinstance(draft_model_config, DFlash2Config):
from aspec.training.dflash2_trainer import DFlash2Trainer

Expand All @@ -67,7 +71,7 @@ def _trainer_class_for_config(draft_model_config):

return Eagle3Trainer
raise ValueError(
"ASpec supports DSpark, DFlash and Llama EAGLE3 draft model configs; "
"ASpec supports DSpark, DFlash, EAGLE3, native MTP and Medusa draft model configs; "
f"got {type(draft_model_config).__name__}"
)

Expand Down
Loading
Loading