diff --git a/aspec/config/draft_algorithm.py b/aspec/config/draft_algorithm.py index 134c03f..d7516fa 100644 --- a/aspec/config/draft_algorithm.py +++ b/aspec/config/draft_algorithm.py @@ -18,6 +18,7 @@ DRAFT_ALGORITHMS = { "NativeMTPDraftModel": "mtp", + "MedusaDraftModel": "medusa", "DFlashDraftModel": "dflash", "DFlash2DraftModel": "dflash2", "DSparkDraftModel": "dspark", @@ -65,7 +66,8 @@ def draft_model_semantics(config) -> dict: "vocab_size", "draft_vocab_size", "hidden_act", "rms_norm_eps", "max_position_embeddings", "rope_theta", "rope_scaling", "rope_parameters", "target_layer_ids", "num_target_layers", "target_num_hidden_layers", - "num_aux_hidden_states", "fc_norm", "norm_output", "mask_token_id", + "num_aux_hidden_states", "medusa_num_heads", "medusa_num_layers", "fc_norm", + "norm_output", "mask_token_id", "lm_head_rank_divisor", "attention_mode", "q_lora_rank", "kv_lora_rank", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim", "block_size", "conv_kernel_size", "conv_group_size", "selector_rank", "selector_top_k", diff --git a/aspec/config/draft_attention.py b/aspec/config/draft_attention.py index c01f45c..f0c1f6f 100644 --- a/aspec/config/draft_attention.py +++ b/aspec/config/draft_attention.py @@ -21,6 +21,11 @@ def resolve_draft_head_layout(raw: Mapping) -> dict: """Resolve the physical Q/KV head layout used by DFlash/DSpark attention.""" + if raw.get("architectures") == ["MedusaDraftModel"]: + return {"type": "heads_only", "query_heads": None, "key_value_heads": None, + "query_heads_per_key_value_head": None, "head_dim": None, + "head_dim_source": "not_applicable"} + def positive_integer(name: str, default: int) -> int: value = raw.get(name, default) if isinstance(value, bool) or not isinstance(value, int) or value <= 0: @@ -115,6 +120,10 @@ def positive_integer(name: str, default: int) -> int: def resolve_draft_attention(raw: Mapping) -> dict: + if raw.get("architectures") == ["MedusaDraftModel"]: + return {"sliding_window": None, "layer_types": [], "compress_ratios": None, + "csa": None, "qsa": None, "context_windows": [], + "window_semantics": "none", "draft_block_attention": "parallel_heads"} layers = raw.get("num_hidden_layers", 1) if isinstance(layers, bool) or not isinstance(layers, int) or layers < 1: raise ValueError("Draft num_hidden_layers must be a positive integer") diff --git a/aspec/config/draft_structure.py b/aspec/config/draft_structure.py index 17f1cd3..26abfb3 100644 --- a/aspec/config/draft_structure.py +++ b/aspec/config/draft_structure.py @@ -23,6 +23,8 @@ def resolve_draft_structure(raw: Mapping) -> dict: from aspec.config.eagle4 import eagle4_structure return eagle4_structure(raw) + if raw.get("architectures") == ["MedusaDraftModel"]: + return {"ffn_type": "heads_only"} ffn_type = raw.get("ffn_type", "dense") if ffn_type not in {"dense", "qwen3_moe", "ling_moe"}: raise ValueError(f"Unsupported Draft ffn_type={ffn_type!r}") diff --git a/aspec/config/medusa_qwen3_8b_draft_config.json b/aspec/config/medusa_qwen3_8b_draft_config.json new file mode 100644 index 0000000..f333aa7 --- /dev/null +++ b/aspec/config/medusa_qwen3_8b_draft_config.json @@ -0,0 +1,18 @@ +{ + "architectures": ["MedusaDraftModel"], + "model_type": "aspec_medusa", + "hidden_size": 4096, + "target_hidden_size": 4096, + "vocab_size": 151936, + "draft_vocab_size": 151936, + "target_num_hidden_layers": 36, + "target_layer_ids": [35], + "num_target_layers": 1, + "num_aux_hidden_states": 1, + "medusa_num_heads": 5, + "medusa_num_layers": 1, + "tie_word_embeddings": false, + "norm_output": false, + "fc_norm": false, + "torch_dtype": "bfloat16" +} \ No newline at end of file diff --git a/aspec/feature_contract.py b/aspec/feature_contract.py index fdb6d02..06bfcc6 100644 --- a/aspec/feature_contract.py +++ b/aspec/feature_contract.py @@ -117,9 +117,10 @@ def build_target_feature_contract( "layout": "sequence", }, } - if drafter_family == "mtp": + if drafter_family in {"mtp", "medusa"}: if len(aux_hidden_state_layers) != 1 or store_last_hidden_states: - raise FeatureContractError("Native MTP requests one post-norm feature, without a duplicate teacher tensor") + raise FeatureContractError( + "Native MTP and Medusa request one post-norm feature, without a duplicate teacher tensor") tensors["hidden_states"] = { "semantic": "target_final_hidden_state", "dtype": "bfloat16", "shape": ["sequence", target_hidden_size], "layout": "sequence_hidden", diff --git a/aspec/inference/sglang_v0521.py b/aspec/inference/sglang_v0521.py index f949236..cc8fc37 100644 --- a/aspec/inference/sglang_v0521.py +++ b/aspec/inference/sglang_v0521.py @@ -593,7 +593,7 @@ def training_engine_kwargs(kwargs, args): "max_sequence_length": args.max_seq_length, "store_last_hidden_states": args.store_last_hidden_states, "direct_prefill": args.target_prefill_memory_optimized, - "final_hidden_as_aux": getattr(args, "draft_algorithm", None) == "mtp", + "final_hidden_as_aux": getattr(args, "draft_algorithm", None) in {"mtp", "medusa"}, "attention_backend": args.attention_backend, **({"mtp": args.aspec_feature_contract["mtp"], "target_model_path": args.target_model_path} if (getattr(args, "aspec_feature_contract", None) or {}).get("mtp") is not None else {}), diff --git a/aspec/models/draft/auto.py b/aspec/models/draft/auto.py index 1b0e5ff..48d6ee6 100644 --- a/aspec/models/draft/auto.py +++ b/aspec/models/draft/auto.py @@ -36,6 +36,7 @@ ) from aspec.models.draft.eagle4 import Eagle4DraftModel from aspec.models.draft.llama3_eagle import LlamaForCausalLMEagle3 +from aspec.models.draft.medusa import MedusaConfig, MedusaDraftModel from aspec.models.draft.mtp import NativeMTPConfig, NativeMTPDraftModel from aspec.utils.logging import logger @@ -46,6 +47,7 @@ class AutoEagle3DraftModel(AutoModelForCausalLMBase): Eagle4Config: Eagle4DraftModel, DeepseekV3Config: Eagle3DeepseekV2ForCausalLM, NativeMTPConfig: NativeMTPDraftModel, + MedusaConfig: MedusaDraftModel, DFlashConfig: DFlashDraftModel, DFlash2Config: DFlash2DraftModel, DSparkConfig: DSparkDraftModel, @@ -68,6 +70,7 @@ class AutoDraftModelConfig: "LlamaForCausalLMEagle4": Eagle4Config, "Eagle3DeepseekV2ForCausalLM": DeepseekV3Config, "NativeMTPDraftModel": NativeMTPConfig, + "MedusaDraftModel": MedusaConfig, "DFlashDraftModel": DFlashConfig, "DFlash2DraftModel": DFlash2Config, "Qwen3DSparkModel": DSparkConfig, diff --git a/aspec/models/draft/medusa.py b/aspec/models/draft/medusa.py index 3e4b20f..6527d00 100644 --- a/aspec/models/draft/medusa.py +++ b/aspec/models/draft/medusa.py @@ -95,9 +95,12 @@ def forward(self, x): class MedusaDraftModel(nn.Module): config_class = MedusaConfig - def __init__(self, config): + def __init__(self, config, attention_backend="sdpa"): super().__init__() + if attention_backend != "sdpa": + raise NotImplementedError("Medusa heads have no attention; use the sdpa backend") self.config = config + self.attention_backend = attention_backend self.medusa_num_heads = config.medusa_num_heads self.medusa_num_layers = config.medusa_num_layers self.hidden_size = config.hidden_size @@ -114,3 +117,9 @@ def __init__(self, config): def forward(self, hidden_states): """[B, S, H] frozen Target features -> [K, B, S, V] per-head logits.""" return torch.stack([head(hidden_states) for head in self.medusa_head], dim=0) + + def load_embedding(self, target_model_path, embedding_key=None): + """Heads-only: the draft reads Target features and owns no token embedding.""" + + def freeze_embedding(self): + """Heads-only: the draft reads Target features and owns no token embedding.""" diff --git a/aspec/planning.py b/aspec/planning.py index 29542fc..fabc01f 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -301,6 +301,34 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: if _select(config, "inference.store_last_hidden_states"): raise PlanCompileError("Native MTP publishes final post-norm features once; set store_last_hidden_states=false") + if family == "medusa": + from aspec.models.draft.medusa import MedusaConfig + + try: + raw = MedusaConfig.from_dict(raw).to_dict() + except (ValueError, TypeError) as exc: + raise PlanCompileError(str(exc)) from exc + backend = _select(config, "training.attention_backend") + if backend != "sdpa": + raise PlanCompileError("Medusa heads have no attention; use training.attention_backend=sdpa") + if _select(config, "training.dflash_packing", False): + raise PlanCompileError("Medusa does not implement sequence packing yet") + if _select(config, "training.draft_context_parallel_size", 1) != 1: + raise PlanCompileError("Medusa currently supports draft_context_parallel_size=1") + if _select(config, "training.micro_batch_size", 1) != 1: + raise PlanCompileError("Medusa currently supports micro_batch_size=1") + if _select(config, "inference.store_last_hidden_states", False): + raise PlanCompileError( + "Medusa publishes final post-norm features once; set store_last_hidden_states=false") + if _select(config, "training.compile_model", False): + raise PlanCompileError("Medusa whole-model compile has not been enabled") + if _select(config, "model.freeze_lm_head", False): + raise PlanCompileError("Medusa heads own trainable vocabulary projections") + if _select(config, "training.eagle3_num_anchors", 0): + raise PlanCompileError("Medusa does not implement anchored training") + if _select(config, "training.ploss_weights") is not None: + raise PlanCompileError("Medusa fixes uniform per-head loss weights") + if family == "eagle4": from aspec.config.eagle4 import Eagle4Config, eagle4_policy @@ -618,6 +646,8 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: "target_hidden_size": raw.get("target_hidden_size"), "target_num_hidden_layers": raw.get("target_num_hidden_layers"), "target_layer_ids": target_layer_ids, + "medusa_num_heads": raw.get("medusa_num_heads"), + "medusa_num_layers": raw.get("medusa_num_layers"), } @@ -1214,7 +1244,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: f"got {declared_target_layers} and {expected_target_layers}" ) training_target_layers = ( - expected_target_layers if draft["family"] in {"eagle3", "eagle4", "mtp"} else getattr( + expected_target_layers if draft["family"] in {"eagle3", "eagle4", "mtp", "medusa"} else getattr( args, "dspark_num_target_layers" if draft["family"] == "dspark" else "dflash_num_target_layers", ) @@ -1227,7 +1257,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: block_size = getattr(args, "dflash_block_size", 16) min_loss_tokens = getattr(args, "min_loss_tokens", 0) - if draft["family"] not in {"eagle3", "eagle4", "mtp"} and min_loss_tokens < 2 * block_size: + if draft["family"] not in {"eagle3", "eagle4", "mtp", "medusa"} and min_loss_tokens < 2 * block_size: raise PlanCompileError( f"{draft['family']} requires dataset.min_loss_tokens >= " f"2 * training.dflash_block_size ({min_loss_tokens} < {2 * block_size})" @@ -1253,6 +1283,11 @@ def _runtime_values(config: DictConfig, draft: Mapping[str, Any], *, source_cont args.ttt_length = args.mtp_steps # common recurrent trainer's metric slots args.ploss_weights = mtp_step_weights(args.mtp_steps, args.mtp_step_weight_beta) args.freeze_lm_head = True + elif args.draft_algorithm == "medusa": + args.ttt_length = draft["medusa_num_heads"] + # The reference sums per-head mean CEs; the trainer applies ttt weights. + args.ploss_weights = [1.0] * args.ttt_length + args.freeze_lm_head = False args.source_context_parallel_size = (args.draft_context_parallel_size if source_context_parallel_size is None else source_context_parallel_size) if args.draft_algorithm in {"eagle3", "eagle4"} and args.ploss_weights is None: @@ -1332,6 +1367,19 @@ def _workload( for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): objective[f"l1_{side}_{name}"] = _select(config, f"training.dspark_l1_{side}_{name}", default) + elif family == "medusa": + objective = { + "family": "medusa", + "loss": "parallel_future_token_ce", + "num_heads": draft["medusa_num_heads"], + "num_layers": draft["medusa_num_layers"], + "prediction_start": 2, + "head_weighting": "sum_of_per_head_mean_ce", + "frozen_shared": ["target_features"], + "verifier_feature_stage": "post_norm", + "vocab_scope": "full_target_vocabulary", + "gradient_checkpointing": _select(config, "training.gradient_checkpointing"), + } else: objective = { "family": family, diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index baea2b4..b2a599b 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -65,6 +65,13 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "step_weights": args.ploss_weights, "loss": "teacher_forced_native_mtp_ce", "frozen_shared": ["embedding", "lm_head"], } + elif getattr(args, "draft_algorithm", None) == "medusa": + topology["training_objective"] = { + "ttt_length": args.ttt_length, + "ploss_weights": args.ploss_weights, + "loss": "parallel_future_token_ce", + "head_weighting": "sum_of_per_head_mean_ce", + } elif getattr(args, "draft_algorithm", None) in {"eagle3", "eagle4"}: topology["training_objective"] = { key: getattr(args, key, None) @@ -96,7 +103,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: if "training_objective" in topology: fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps", "max_grad_norm", "seed") - fields += (("freeze_lm_head",) if args.draft_algorithm in {"eagle3", "eagle4", "mtp"} + fields += (("freeze_lm_head",) if args.draft_algorithm in {"eagle3", "eagle4", "mtp", "medusa"} else ("deterministic_anchor_sampling",)) for key in fields: if hasattr(args, key): diff --git a/aspec/training/medusa_trainer.py b/aspec/training/medusa_trainer.py new file mode 100644 index 0000000..1986493 --- /dev/null +++ b/aspec/training/medusa_trainer.py @@ -0,0 +1,44 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Heads-only Medusa uses the existing optimizer/checkpoint/ownership training path.""" + +from types import SimpleNamespace + +from aspec.models.medusa import MedusaModel +from aspec.training.eagle3_trainer import Eagle3Trainer +from aspec.training.trainer import Trainer + + +class MedusaTrainer(Eagle3Trainer): + def _create_training_model(self, draft_model): + return MedusaModel( + draft_model, + gradient_checkpointing=getattr(self.args, "gradient_checkpointing", False), + ) + + def _prepare_algorithm_training(self, draft_model): + if getattr(self.args, "freeze_lm_head", False): + raise ValueError("Medusa heads own trainable vocabulary projections") + + def _init_target_lm_head(self, target_model_path): + # CE needs no teacher distribution. The trainer's unconditional + # weight/norm reads at init only need an object; reuse an existing head. + self.target_lm_head = SimpleNamespace( + lm_head=self.draft_model.medusa_head[0][-1], norm=None) + + def get_feature_contract_status(self): + status = dict(Trainer.get_feature_contract_status(self)) + status["medusa_num_heads"] = getattr(self.args, "ttt_length", None) + status["medusa_num_layers"] = getattr( + getattr(self, "draft_model", None), "medusa_num_layers", None) + return status + + def _forward(self, batch): + return self.model( + input_ids=batch["input_ids"].cuda(), + hidden_states=batch["hidden_states"].cuda(), + loss_mask=batch["loss_mask"].squeeze(-1).cuda() + if batch["loss_mask"].ndim == 3 else batch["loss_mask"].cuda(), + attention_mask=batch["attention_mask"].cuda(), + ) diff --git a/aspec/training/trainer_actor.py b/aspec/training/trainer_actor.py index 2587568..7b729a8 100644 --- a/aspec/training/trainer_actor.py +++ b/aspec/training/trainer_actor.py @@ -48,6 +48,10 @@ def _trainer_class_for_config(draft_model_config): from aspec.training.mtp_trainer import MTPTrainer return MTPTrainer + if getattr(draft_model_config, "architectures", None) == ["MedusaDraftModel"]: + from aspec.training.medusa_trainer import MedusaTrainer + + return MedusaTrainer if isinstance(draft_model_config, DFlash2Config): from aspec.training.dflash2_trainer import DFlash2Trainer @@ -67,7 +71,7 @@ def _trainer_class_for_config(draft_model_config): return Eagle3Trainer raise ValueError( - "ASpec supports DSpark, DFlash and Llama EAGLE3 draft model configs; " + "ASpec supports DSpark, DFlash, EAGLE3, native MTP and Medusa draft model configs; " f"got {type(draft_model_config).__name__}" ) diff --git a/configs/examples/medusa_qwen3_8b_4k_tcp.yaml b/configs/examples/medusa_qwen3_8b_4k_tcp.yaml new file mode 100644 index 0000000..046d753 --- /dev/null +++ b/configs/examples/medusa_qwen3_8b_4k_tcp.yaml @@ -0,0 +1,87 @@ +# Supplementary configuration example; see the support matrix for supported workflows. +# Single-node heads-only Medusa (K=5) on a frozen Qwen3-8B Target: one Target GPU +# plus one Draft GPU, 4K, CP1, no packing, SDPA, Store/TCP transport (no NVLink). +# Uses the bundled conversations; the short window validates the execution path, +# not convergence, acceptance or decoding quality. + +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL} + native_context_length: 32768 + trust_remote_code: false + draft_model_config: aspec/config/medusa_qwen3_8b_draft_config.json + +dataset: + train_data_path: ../../examples/data/sample_conversations.jsonl + eval_data_path: null + chat_template: qwen3-instruct + min_loss_tokens: 1 + shuffle_dataset: false + +training: + attention_backend: sdpa + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 4 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + max_grad_norm: 1.0 + max_seq_length: 4096 + num_epochs: 100 + num_train_steps: 8 + lr_total_steps: 8 + seed: 42 + training_num_gpus_per_node: 2 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + prefetch_depth: 1 + save_interval: 0 + save_per_epoch: false + save_final_checkpoint: true + warmup_ratio: 0.04 + dflash_packing: false + gradient_checkpointing: true + +inference: + inference_engine_type: sgl + store_last_hidden_states: false + target_prefill_memory_optimized: false + inference_num_gpus: 1 + inference_num_gpus_per_engine: 1 + inference_num_gpus_per_node: 2 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 12 + inference_batch_size: 2 + sglang: + tp_size: 1 + mem_fraction_static: 0.5 + +mooncake: + master_server_address: null + metadata_server: null + data_plane: store + protocol: tcp + enable_gpu_direct: false + allow_host_fallback: false + global_segment_size: 32GB + local_buffer_size: 8GB + host_buffer_size: 1GB + async_put_pool_size: 1 + enable_hard_pin: true + max_seq_len: 4096 + hidden_dim: 4096 + +logging: + report_to: none + +debug: + enable_perf_metrics: true + +output_dir: outputs/medusa_qwen3_8b_4k_tcp +cache_dir: cache/medusa_qwen3_8b_4k_tcp \ No newline at end of file diff --git a/docs/support.md b/docs/support.md index fe553a6..6e6c0c8 100644 --- a/docs/support.md +++ b/docs/support.md @@ -12,6 +12,7 @@ and tests do not imply CPU training support. | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery; SDPA CP1, MBS1, non-packing, REPLICATE | No packing/CP, warmstart, export or consumer acceptance yet; `K=5` on Qwen3-8B is about 3.2B trainable parameters; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | MTP-conditioned DSpark / DFlash | Target-side frozen native Qwen3.5 features, token/hidden dual-anchor training, exact recovery and strict native assets; legacy Draft-side assets retained | Dense full GQA, CP1, MBS1, REPLICATE, one node, non-packing, decay objective; Target capture PP1/unquantized; no generic serving consumer | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 9ebe89d..548a8dd 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -135,3 +135,13 @@ python -m tools.stitch_native_mtp --target /path/to/Qwen3.5-9B \ The original Target is not overwritten. Official SGLang consumption selects `EAGLE`, not a `MTP` flag. A format manifest does not replace testing the particular exported asset with its consumer. + +## Medusa heads-only + +`aspec/config/medusa_qwen3_8b_draft_config.json` trains K parallel heads on the frozen Target's +post-norm final state; head `i` at position `t` is supervised by token `t+2+i`. The objective is +the sum of per-head mean cross-entropies (assistant tokens only), and the heads own trainable +full-vocabulary projections. Start from `configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (SDPA, +CP1, MBS1, non-packing, REPLICATE, Store/TCP). `K=5` on Qwen3-8B is about 3.2B trainable +parameters; measure memory fit before long runs. Export, a maintained recipe, per-head held-out +observations and candidate-tree acceptance remain future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md new file mode 100644 index 0000000..e2f6cf4 --- /dev/null +++ b/docs/user-guide/medusa.md @@ -0,0 +1,48 @@ +# Medusa heads-only Draft + +Heads-only Medusa-1 training on frozen Target features, implemented from +[FasterDecoding/Medusa](https://github.com/FasterDecoding/Medusa) at +`e2a5d20c048a9b0a4092e6933c34313687422518` (Apache-2.0; the adapted source is based on +`tatsu-lab/stanford_alpaca` and adapts FastChat code, with those notices retained). The model and +loss landed first; this page describes the current training scope. + +## Reference and configuration + +- Draft config: `aspec/config/medusa_qwen3_8b_draft_config.json` (Qwen3-8B Target, `K=5` heads). +- Example: `configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (one Target GPU + one Draft GPU, 4K, + SDPA, CP1, non-packing, REPLICATE, Store/TCP transport; no NVLink required). +- The heads are the only trainable parameters; the Target is frozen and its post-norm final + state is the only feature the Draft consumes. + +## Inputs and prediction semantics + +- Each head is `medusa_num_layers` residual blocks (`x + SiLU(Linear(x))` with a + zero-initialized weight and the default bias) followed by a bias-free + `hidden_size -> vocab_size` projection. The reference repeats one shared residual block per + head depth; that weight sharing is preserved. +- Head `i` at position `t` is supervised by the token at `t+2+i` (the reference trainer's + offset). Supervision is assistant-only through the existing loss mask. +- The objective is the sum of per-head mean cross-entropies over the full Target vocabulary. + The trainer applies uniform per-head weights (`training.ttt_length = K`, + `ploss_weights = [1, 1, ...]`), so the aggregate equals the reference sum. +- No token embedding, no Draft attention, no teacher logits. The shared-embedding and + shared-head conventions of native MTP do not apply: every head owns its vocabulary projection. + +## Supported scope + +The current path fails closed outside the following: + +- `training.attention_backend: sdpa`; CP1 (`draft_context_parallel_size: 1`); + `micro_batch_size: 1`; `dflash_packing: false`; `deterministic_anchor_sampling` is unused. +- `inference.store_last_hidden_states: false` (the single post-norm slot is the feature). +- `fsdp_strategy: REPLICATE`, one node. Anchored training, MTP conditioning and whole-model + compile are rejected. + +## Capacity + +One head holds `hidden_size^2 + hidden_size * vocab_size` parameters (about 0.64B for Qwen3-8B), +so `K=5` is about 3.2B trainable parameters. Bfloat16 weights, gradients, f32 master weights and +Adam state are roughly 51 GB in REPLICATE, which does not fit one 48 GB device for the listed +example. Size the device count for your hardware, use `fsdp_strategy: SHARD` across independent +Draft GPUs when memory requires it, and measure the actual fit. Export, a maintained recipe, +per-head held-out observations and candidate-tree acceptance are not part of this scope. diff --git a/tests/cpu/test_medusa_integration.py b/tests/cpu/test_medusa_integration.py new file mode 100644 index 0000000..9438155 --- /dev/null +++ b/tests/cpu/test_medusa_integration.py @@ -0,0 +1,241 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Medusa integration contracts: registration, Plan, features, dispatch and resume.""" + +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from aspec.config.draft_algorithm import draft_model_semantics, resolve_draft_algorithm +from aspec.config.draft_attention import resolve_draft_attention, resolve_draft_head_layout +from aspec.config.draft_structure import resolve_draft_structure +from aspec.config.train_config import load_config +from aspec.feature_contract import FeatureContractError, build_target_feature_contract +from aspec.models.draft.auto import AutoDraftModelConfig, AutoEagle3DraftModel +from aspec.models.draft.medusa import MedusaConfig, MedusaDraftModel +from aspec.planning import ( + PlanCompileError, + _load_draft_descriptor, + _runtime_values, + compile_execution_plan, +) +from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, +) +from aspec.training.medusa_trainer import MedusaTrainer +from aspec.training.trainer_actor import _trainer_class_for_config + +_REPO_ROOT = Path(__file__).parents[2] +_EXAMPLE = _REPO_ROOT / "configs" / "examples" / "medusa_qwen3_8b_4k_tcp.yaml" +_DRAFT_JSON = _REPO_ROOT / "aspec" / "config" / "medusa_qwen3_8b_draft_config.json" + +_ILLEGAL_OVERRIDES = [ + ("training.dflash_packing=true", PlanCompileError), + ("training.draft_context_parallel_size=2", NotImplementedError), + ("training.micro_batch_size=2", PlanCompileError), + ("training.attention_backend=fa4", PlanCompileError), + ("training.attention_backend=cudnn_flex_attention", PlanCompileError), + ("inference.store_last_hidden_states=true", PlanCompileError), + ("training.compile_model=true", PlanCompileError), + ("model.freeze_lm_head=true", PlanCompileError), + ("training.ploss_weights=[1.0,1.0]", PlanCompileError), + ("training.eagle3_num_anchors=1", PlanCompileError), +] + + +def _tiny_config(**overrides): + raw = dict(architectures=["MedusaDraftModel"], model_type="aspec_medusa", + hidden_size=16, vocab_size=32, target_hidden_size=16, + target_num_hidden_layers=4, medusa_num_heads=3, medusa_num_layers=1) + raw.update(overrides) + return AutoDraftModelConfig.from_dict(raw) + + +def _fake_qwen_target(tmp_path): + target = tmp_path / "Qwen3-8B-test" + target.mkdir() + (target / "config.json").write_text(json.dumps({ + "model_type": "qwen3", + "architectures": ["Qwen3ForCausalLM"], + "hidden_size": 4096, + "vocab_size": 151936, + "num_hidden_layers": 36, + "max_position_embeddings": 40960, + }), encoding="utf-8") + return target + + +def _config(tmp_path, *overrides): + args = [f"model.target_model_path={_fake_qwen_target(tmp_path)}", *overrides] + return load_config(config_path=str(_EXAMPLE), cli_args=args), args + + +def test_registration_resolves_medusa(): + assert resolve_draft_algorithm({"architectures": ["MedusaDraftModel"]}) == "medusa" + shipped = AutoDraftModelConfig.from_file(str(_DRAFT_JSON)) + assert isinstance(shipped, MedusaConfig) + assert shipped.medusa_num_heads == 5 + assert shipped.medusa_num_layers == 1 + assert shipped.target_layer_ids == [35] + assert shipped.num_aux_hidden_states == 1 + + tiny = _tiny_config() + model = AutoEagle3DraftModel.from_config(tiny, attention_backend="sdpa") + assert isinstance(model, MedusaDraftModel) + assert len(model.medusa_head) == 3 + with pytest.raises(NotImplementedError): + AutoEagle3DraftModel.from_config(tiny, attention_backend="fa4") + assert _trainer_class_for_config(tiny) is MedusaTrainer + semantics = draft_model_semantics(tiny) + assert semantics["medusa_num_heads"] == 3 + assert semantics["medusa_num_layers"] == 1 + + +def test_heads_only_policies(): + raw = _tiny_config().to_dict() + attention = resolve_draft_attention(raw) + assert attention["layer_types"] == [] + assert attention["context_windows"] == [] + assert attention["draft_block_attention"] == "parallel_heads" + layout = resolve_draft_head_layout(raw) + assert layout["type"] == "heads_only" + assert layout["head_dim"] is None + assert resolve_draft_structure(raw) == {"ffn_type": "heads_only"} + + +def test_feature_contract_requests_one_post_norm_state(): + contract = build_target_feature_contract( + drafter_family="medusa", target_hidden_size=4096, target_num_hidden_layers=36, + aux_hidden_state_layers=[35], store_last_hidden_states=False) + hidden = contract["tensors"]["hidden_states"] + assert hidden["semantic"] == "target_final_hidden_state" + assert hidden["value_stage"] == "post_norm" + assert "last_hidden_states" not in contract["tensors"] + with pytest.raises(FeatureContractError): + build_target_feature_contract( + drafter_family="medusa", target_hidden_size=4096, target_num_hidden_layers=36, + aux_hidden_state_layers=[35], store_last_hidden_states=True) + with pytest.raises(FeatureContractError): + build_target_feature_contract( + drafter_family="medusa", target_hidden_size=4096, target_num_hidden_layers=36, + aux_hidden_state_layers=[34, 35], store_last_hidden_states=False) + + +def test_plan_compiles_with_medusa_objective(tmp_path): + config, args = _config(tmp_path) + plan = compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args).to_dict() + draft = plan["workload"]["models"]["draft"] + assert draft["family"] == "medusa" + assert draft["structure"] == {"ffn_type": "heads_only"} + assert draft["attention"]["head_layout"]["type"] == "heads_only" + objective = plan["workload"]["objective"] + assert objective["num_heads"] == 5 + assert objective["num_layers"] == 1 + assert objective["loss"] == "parallel_future_token_ce" + assert objective["head_weighting"] == "sum_of_per_head_mean_ce" + assert objective["prediction_start"] == 2 + features = plan["workload"]["features"] + assert features["aux_hidden_state_layers"] == [35] + assert features["tensors"]["hidden_states"]["semantic"] == "target_final_hidden_state" + assert "last_hidden_states" not in features["tensors"] + + runtime = _runtime_values(config, _load_draft_descriptor(config)) + assert runtime["draft_algorithm"] == "medusa" + assert runtime["ttt_length"] == 5 + assert runtime["ploss_weights"] == [1.0] * 5 + assert runtime["freeze_lm_head"] is False + + +@pytest.mark.parametrize("override, error", _ILLEGAL_OVERRIDES) +def test_plan_rejects_unsupported_medusa_combinations(tmp_path, override, error): + target = f"model.target_model_path={_fake_qwen_target(tmp_path)}" + args = [target, override] + with pytest.raises(error): + config = load_config(config_path=str(_EXAMPLE), cli_args=args) + compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args) + + +def test_producer_publishes_final_state_for_medusa(monkeypatch): + for name in ("sglang", "sglang.srt", "sglang.srt.model_executor", + "mooncake", "mooncake.store"): + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + sys.modules["sglang.srt.model_executor"].hook_manager = SimpleNamespace( + SUPPORTS_MODEL_HOOK_SETUP=True) + sys.modules["mooncake.store"].MooncakeDistributedStore = object + from aspec.inference.sglang_v0521 import training_engine_kwargs + + args = SimpleNamespace( + train_with_decode=False, draft_context_parallel_size=1, + draft_context_shard_alignment=1, max_seq_length=4096, + store_last_hidden_states=False, target_prefill_memory_optimized=False, + attention_backend="sdpa", mooncake_hidden_dim=4096, draft_algorithm="medusa") + kwargs = {"aux_hidden_state_layer_ids": [35]} + published = training_engine_kwargs(kwargs, args)["forward_hooks"][-1]["config"] + assert published["final_hidden_as_aux"] is True + args.draft_algorithm = "mtp" + assert training_engine_kwargs(kwargs, args)["forward_hooks"][-1]["config"][ + "final_hidden_as_aux"] is True + args.draft_algorithm = "dspark" + assert training_engine_kwargs(kwargs, args)["forward_hooks"][-1]["config"][ + "final_hidden_as_aux"] is False + + +def _trainer_args(config, **overrides): + values = dict( + draft_algorithm="medusa", ttt_length=3, ploss_weights=[1.0, 1.0, 1.0], + draft_model_config_obj=config, world_size=2, freeze_lm_head=False, + max_seq_length=4096, micro_batch_size=1, draft_accumulation_steps=1, + max_grad_norm=1.0, seed=42, draft_context_parallel_size=1, + training_num_nodes=1, training_num_gpus_per_node=2, attention_backend="sdpa", + fsdp_strategy="REPLICATE", sp_ulysses_size=1, sp_ring_size=1, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def test_target_head_binding_is_rng_free_and_reuses_an_existing_head(): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + draft = MedusaDraftModel(_tiny_config()) + trainer.draft_model = draft + before = torch.get_rng_state().clone() + trainer._init_target_lm_head("unused") + assert trainer.target_lm_head.lm_head is draft.medusa_head[0][-1] + assert trainer.target_lm_head.norm is None + assert torch.equal(torch.get_rng_state(), before) + + +def test_frozen_head_is_rejected(): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + trainer.args = SimpleNamespace(freeze_lm_head=True) + with pytest.raises(ValueError): + trainer._prepare_algorithm_training(MedusaDraftModel(_tiny_config())) + + +def test_checkpoint_topology_pins_head_count(): + config = _tiny_config() + args = _trainer_args(config) + current = _logical_training_topology(args, 2) + objective = current["training_objective"] + assert objective["ttt_length"] == 3 + assert objective["ploss_weights"] == [1.0, 1.0, 1.0] + assert objective["head_weighting"] == "sum_of_per_head_mean_ce" + assert current["draft_ffn_policy"] == {"ffn_type": "heads_only"} + + metadata = {"world_size": 2, + "training_topology": json.loads(json.dumps(current))} + _validate_exact_resume_topology(args, metadata) + changed = json.loads(json.dumps(metadata)) + changed["training_topology"]["training_objective"]["ttt_length"] = 5 + with pytest.raises(RuntimeError): + _validate_exact_resume_topology(args, changed)