diff --git a/examples/llm_finetune/minimax_m2/minimax_m2.7_hellaswag_lora.yaml b/examples/llm_finetune/minimax_m2/minimax_m2.7_hellaswag_lora.yaml index 787cbc4def..57c68b5e1b 100644 --- a/examples/llm_finetune/minimax_m2/minimax_m2.7_hellaswag_lora.yaml +++ b/examples/llm_finetune/minimax_m2/minimax_m2.7_hellaswag_lora.yaml @@ -125,9 +125,33 @@ ci: # Use Transformers' built-in MiniMax M2 implementation for the HF+PEFT # reload; the checkpoint's remote code targets an older Transformers API. trust_remote_code: false - # Keep the cross-framework gates blocking: the current built-in-HF mismatch - # must be fixed or explicitly proven to be an invalid reference before it is skipped. tokenizer_name: MiniMaxAI/MiniMax-M2.7 hf_device_map_auto: true + # Measured envelope for the two cross-framework gates (AMINT-286). The + # residual AM-vs-HF divergence is knife-edge routing amplifying bf16 + # kernel noise, not an implementation defect: the checkpoint's fp32 + # e_score_correction_bias lattices put top-8 selection inside the + # kernel-noise band (68% of tokens flip an expert within five layers, + # pipeline 64182587), and the HF reference against itself (eager vs sdpa + # attention, 62-layer sweep on real weights) diverges at the same order + # (mean KL 0.237 vs AutoModel's 0.320). Gate the full 2048-token document + # at the measured envelope (three scoped runs: mean KL 0.086-0.094, + # p95 0.346-0.375, cos 0.964-0.965); real conversion or model-math bugs + # stay loud (the repaired rope-reference bug measured mean KL 13.1). + parity_threshold_overrides: + source_load: + mean_kl: 0.15 + p95_kl: 0.5 + cosine_similarity: 0.95 + hf_reload: + mean_kl: 0.15 + p95_kl: 0.5 + cosine_similarity: 0.95 + # Restored state and the first resumed forward are exact; three scoped-CI + # runs measured 0.012-0.020 routed-MoE loss drift at continuation steps 6-7 + # versus the standard 0.0117 allowance (pipelines 63311388, 63340676, + # 64093591). Match the routed hybrid-MoE precedent and use the shared + # relaxed resume envelope; every logit gate stays standard. + resume_tolerance_profile: relaxed dataset.num_samples_limit: 500 validation_dataset.num_samples_limit: 500 diff --git a/examples/vlm_finetune/minimax_m3/minimax_m3_vl_lora_pp4ep8_8node.yaml b/examples/vlm_finetune/minimax_m3/minimax_m3_vl_lora_pp4ep8_8node.yaml index d3b7e5d698..5018fb16ba 100644 --- a/examples/vlm_finetune/minimax_m3/minimax_m3_vl_lora_pp4ep8_8node.yaml +++ b/examples/vlm_finetune/minimax_m3/minimax_m3_vl_lora_pp4ep8_8node.yaml @@ -164,4 +164,25 @@ wandb: ci: recipe_owner: athitten nodes: 8 - time: "00:20:00" + # The robustness matrix (train/save, reload, resume) measured 1028s of + # phase time on the first scoped run (job 412098422, ~32 min including + # CI-side setup outside the Slurm window); 45 min covers it with headroom. + time: "00:45:00" + checkpoint_robustness: + # PP=4 with pp_microbatch_size=1 needs at least four pipeline microbatches; + # dp = 64/4 = 16, so global = 4 * 16 with one grad-accum step. + step_scheduler.local_batch_size: 4 + step_scheduler.global_batch_size: 64 + tokenizer_name: MiniMaxAI/MiniMax-M3 + # The 427B (854 GiB bf16) vanilla reference does not fit rank 0's eight + # GPUs (uncapped device_map OOMs on the fused-expert concat; capped, it + # CPU-offloads ~400 GiB and idles all 64 GPUs for hours per run). Skip + # both vanilla-HF phases entirely: MiniMax-M2.7 carries the family's + # blocking cross-framework parity gates with the same measured knife-edge + # router lattice (AMINT-286), while this recipe keeps the AutoModel-side + # gates blocking (train/save, bitwise reload, resume) for the VL wrapper + # and mixed dense/sparse decoder. + skip_source_load_parity: true + skip_hf_reload: true + # Routed hybrid-MoE resume precedent (MiniMax M2.7, Nemotron chat). + resume_tolerance_profile: relaxed diff --git a/nemo_automodel/components/models/minimax_m2/model.py b/nemo_automodel/components/models/minimax_m2/model.py index f80a18ea60..15123f97ec 100644 --- a/nemo_automodel/components/models/minimax_m2/model.py +++ b/nemo_automodel/components/models/minimax_m2/model.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any, Union import torch @@ -98,6 +98,14 @@ def __init__( moe_overrides: dict | None = None, ): super().__init__() + # Released MiniMax-M2 checkpoints store the router gate weight in fp32, + # and the HF reference projects with hidden_states.to(weight.dtype), so + # the checkpoint-faithful router runs an fp32 projection, fp32 scoring, + # and fp32 selected weights. Keep that default while preserving an + # explicit backend override (see AMINT-286; ERNIE follows the same + # pattern for its fp32 router). + if backend.gate_precision is None: + backend = replace(backend, gate_precision=torch.float32) self.backend = backend self.config = config if moe_config is not None and moe_overrides is not None: @@ -133,6 +141,14 @@ def __init__( expert_activation="swiglu", softmax_before_topk=(score_func == "softmax"), force_e_score_correction_bias=True, + # The HF reference returns selected weights in the fp32 router + # logits dtype; keep them fp32 through the expert combine. + router_weights_fp32=True, + # The checkpoint stores the gate weight in fp32; allocate it fp32 so + # every construction path (including meta-device init before FSDP + # sharding) keeps the gate's dtype group uniform with its fp32 + # correction-bias buffer. + gate_dtype=torch.float32, dtype=model_dtype, ) if moe_overrides: @@ -229,7 +245,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class MiniMaxM2ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY - _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] + _keep_in_fp32_modules_strict = ["mlp.gate.weight", "mlp.gate.e_score_correction_bias"] @dataclass(frozen=True) class ModelCapabilities: diff --git a/nemo_automodel/components/models/minimax_m3_vl/model.py b/nemo_automodel/components/models/minimax_m3_vl/model.py index fd2e1f1b20..d608377ed7 100644 --- a/nemo_automodel/components/models/minimax_m3_vl/model.py +++ b/nemo_automodel/components/models/minimax_m3_vl/model.py @@ -96,6 +96,11 @@ def build_moe_config(config: Any, dtype: torch.dtype) -> MoEConfig: activation_limit=float(getattr(config, "swiglu_limit", 7.0)), softmax_before_topk=False, force_e_score_correction_bias=bool(getattr(config, "use_routing_bias", True)), + # Released MiniMax-M3 checkpoints store the router gate weight in fp32 + # (same 1e-3-quantized correction-bias lattice as MiniMax-M2.7); allocate + # it fp32 so every construction path keeps the gate's FSDP dtype group + # uniform with its fp32 bias buffer (AMINT-286 pattern). + gate_dtype=torch.float32, dtype=dtype, ) @@ -264,7 +269,7 @@ class MiniMaxM3SparseForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMix tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY - _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] + _keep_in_fp32_modules_strict = ["mlp.gate.weight", "mlp.gate.e_score_correction_bias"] # The state-dict adapter loads every tensor from the checkpoint, so skip HF # random init on load (also avoids DTensor-collective hangs under sharding/PP). @@ -389,7 +394,7 @@ class MiniMaxM3SparseForConditionalGeneration(HFCheckpointingMixin, nn.Module, M # (vision_encoder.py) fp32 — the bf16 cast would otherwise round it and degrade # vision RoPE (see llama/rope_utils.py). _keep_in_fp32_modules = ["rotary_emb", "inv_freq"] - _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] + _keep_in_fp32_modules_strict = ["mlp.gate.weight", "mlp.gate.e_score_correction_bias"] _pp_keep_self_forward: bool = True mtp_outputs_are_logits = True # Opt into context parallelism on the SDPA attention backend (M3's block-sparse DSA diff --git a/nemo_automodel/components/moe/config.py b/nemo_automodel/components/moe/config.py index 6255cb8150..e1bf85b235 100644 --- a/nemo_automodel/components/moe/config.py +++ b/nemo_automodel/components/moe/config.py @@ -56,6 +56,11 @@ class MoEConfig: router_weights_fp32: bool = False router_weight_uses_score_correction_bias: bool = False dtype: str | torch.dtype = torch.bfloat16 + # Storage dtype for the router gate parameters. None inherits ``dtype``. + # Models whose checkpoints store the gate in fp32 (e.g. MiniMax-M2) set + # this so the gate is fp32 from allocation on every construction path, + # keeping FSDP dtype groups uniform with the fp32 correction-bias buffer. + gate_dtype: str | torch.dtype | None = None shared_expert_gate: bool = False shared_expert_inter_dim: int | None = None shared_expert_activation: str = "swiglu" # Activation for shared experts ("swiglu" or "relu2") @@ -74,6 +79,8 @@ def expert_dim(self) -> int: def __post_init__(self): if isinstance(self.dtype, str): self.dtype = dtype_from_str(self.dtype, default=torch.bfloat16) + if isinstance(self.gate_dtype, str): + self.gate_dtype = dtype_from_str(self.gate_dtype, default=torch.bfloat16) @dataclass diff --git a/nemo_automodel/components/moe/layers.py b/nemo_automodel/components/moe/layers.py index 30e3a39773..2afe823ec9 100644 --- a/nemo_automodel/components/moe/layers.py +++ b/nemo_automodel/components/moe/layers.py @@ -266,16 +266,24 @@ def __init__( self.aux_loss_coeff = config.aux_loss_coeff self.norm_topk_prob = config.norm_topk_prob self.gate_precision = gate_precision + # Score arithmetic always has a concrete dtype: an explicit + # gate_precision wins, otherwise fp32 (the shared default of both + # score branches). gate_precision itself stays tri-state because + # None means "project in the input's runtime dtype", which several + # reference routers require (AM-821: Qwen3-MoE, GPT-OSS, DSV4, + # Laguna project in bf16 while scoring in fp32). + self.score_dtype = gate_precision if gate_precision is not None else torch.float32 if self.bias_update_factor > 0: assert self.train_gate, "Require train_gate to be set to True to apply the bias update" + gate_dtype = config.gate_dtype or config.dtype self.weight = nn.Parameter( - torch.empty(config.n_routed_experts, config.dim, dtype=config.dtype), requires_grad=self.train_gate + torch.empty(config.n_routed_experts, config.dim, dtype=gate_dtype), requires_grad=self.train_gate ) if config.router_bias: self.bias = nn.Parameter( - torch.empty(config.n_routed_experts, dtype=config.dtype), requires_grad=self.train_gate + torch.empty(config.n_routed_experts, dtype=gate_dtype), requires_grad=self.train_gate ) else: self.bias = None @@ -405,7 +413,7 @@ def _route_scores(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tenso if self.score_func == "softmax": if self.softmax_before_topk: - scores = scores.softmax(dim=-1, dtype=self.gate_precision or torch.float32) + scores = scores.softmax(dim=-1, dtype=self.score_dtype) original_scores = scores indices = torch.topk(scores, k=self.topk, dim=-1)[1] indices = replay_selection(self.router_replay, indices) @@ -418,14 +426,14 @@ def _route_scores(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tenso # replayed experts. Skipped (zero overhead) on the default path. values = scores.gather(1, replayed) indices = replayed - weights = values.softmax(dim=1, dtype=self.gate_precision or torch.float32) + weights = values.softmax(dim=1, dtype=self.score_dtype) # Use full softmax for aux_loss so P_i represents proper probabilities. # Raw logits can be negative, causing aux_loss to diverge negative. - original_scores = scores.softmax(dim=-1, dtype=self.gate_precision or torch.float32) + original_scores = scores.softmax(dim=-1, dtype=self.score_dtype) elif self.score_func == "softmax_with_bias": # softmax first, then add bias for expert selection, # group routing on biased scores, final weights from unbiased softmax scores. - scores = scores.softmax(dim=-1, dtype=self.gate_precision or torch.float32) + scores = scores.softmax(dim=-1, dtype=self.score_dtype) original_scores = scores # Add correction bias for expert SELECTION only @@ -458,7 +466,11 @@ def _route_scores(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tenso indices = replay_selection(self.router_replay, indices) weights = original_scores.gather(1, indices) elif self.score_func == "sigmoid_with_bias": - scores = scores.sigmoid() + # Score in fp32 like the softmax path: HF sigmoid-router references + # compute sigmoid(logits.float()), and bf16 sigmoid quantizes scores + # at ~2e-3 — enough to flip knife-edge e_score_correction_bias + # selections (AMINT-286). + scores = torch.sigmoid(scores.to(dtype=self.score_dtype)) original_scores = scores scores_for_choice = scores @@ -479,7 +491,8 @@ def _route_scores(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tenso indices = replay_selection(self.router_replay, indices) weights = original_scores.gather(1, indices) else: - scores = scores.sigmoid() + # Score in fp32 like the softmax path (see sigmoid_with_bias above). + scores = torch.sigmoid(scores.to(dtype=self.score_dtype)) original_scores = scores # Add correction bias to balance tokens across gates. diff --git a/tests/functional_tests/checkpoint_robustness/test_checkpoint_robustness_llm.py b/tests/functional_tests/checkpoint_robustness/test_checkpoint_robustness_llm.py index f0bee9e289..71dd55691d 100644 --- a/tests/functional_tests/checkpoint_robustness/test_checkpoint_robustness_llm.py +++ b/tests/functional_tests/checkpoint_robustness/test_checkpoint_robustness_llm.py @@ -343,6 +343,42 @@ def _load_hf_fp8_dequantized_config( return config +def _repair_legacy_partial_rotary_config(config) -> bool: + """Restore a legacy partial-rotary spec dropped by newer Transformers configs. + + Checkpoints such as MiniMax-M2.* express partial RoPE only through the + legacy ``rotary_dim`` config field. Transformers 5.x in-tree configs keep + ``rotary_dim`` as a plain attribute while their models read only + ``rope_parameters["partial_rotary_factor"]``, so the vanilla reference + silently rotates the full head dimension with the wrong frequency ladder + and becomes a deterministic but invalid reference (AMINT-286). Derive the + missing factor as ``rotary_dim / head_dim``. + + Args: + config: Loaded HF config for the vanilla reference model. + + Returns: + True when the config's rope parameters were repaired; False when the + config has no legacy spec or already carries a partial factor. + """ + rotary_dim = getattr(config, "rotary_dim", None) + head_dim = getattr(config, "head_dim", None) + if not rotary_dim or not head_dim or rotary_dim == head_dim: + return False + rope_parameters = getattr(config, "rope_parameters", None) + if isinstance(rope_parameters, dict): + if rope_parameters.get("partial_rotary_factor"): + return False + rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim + return True + if rope_parameters is not None and hasattr(rope_parameters, "partial_rotary_factor"): + if rope_parameters.partial_rotary_factor: + return False + rope_parameters.partial_rotary_factor = rotary_dim / head_dim + return True + return False + + def _is_nemo_owned_config(config) -> bool: """Return True when a config object is an AutoModel component config.""" return type(config).__module__.startswith("nemo_automodel") @@ -368,6 +404,14 @@ def _replace_nemo_owned_reference_config( (AMINT-288). Resolve the checkpoint's own config class from its ``auto_map`` instead, preserving a load-time FP8 ``dequantize`` request. + The same registrations also shadow in-tree config classes: for built-in + references (``trust_remote_code=False``), AutoConfig pairs the + AutoModel-owned config with the in-tree model, which then crashes on + attribute contracts the local class does not carry (the in-tree + minimax_m3_vl vision tower reads ``vision_config.temporal_patch_size``). + Resolve the in-tree class from Transformers' own name table, which + registration cannot shadow. + Args: config: Config resolved for the vanilla reference load. pretrained_model_name_or_path: Checkpoint the reference loads from. @@ -378,15 +422,9 @@ def _replace_nemo_owned_reference_config( Returns: Tuple of the faithful config and whether a replacement happened. """ - if not trust_remote_code or not _is_nemo_owned_config(config): - return config, False - auto_map = getattr(config, "auto_map", None) or {} - class_reference = auto_map.get("AutoConfig") - if not class_reference: + if not _is_nemo_owned_config(config): return config, False - from transformers.dynamic_module_utils import get_class_from_dynamic_module - load_kwargs: dict[str, str | bool] = { "local_files_only": os.environ.get("HF_HUB_OFFLINE", "0") == "1", } @@ -394,7 +432,23 @@ def _replace_nemo_owned_reference_config( load_kwargs["revision"] = revision if token is not None: load_kwargs["token"] = token - config_cls = get_class_from_dynamic_module(class_reference, pretrained_model_name_or_path, **load_kwargs) + + config_cls = None + auto_map = getattr(config, "auto_map", None) or {} + class_reference = auto_map.get("AutoConfig") + if trust_remote_code and class_reference: + from transformers.dynamic_module_utils import get_class_from_dynamic_module + + config_cls = get_class_from_dynamic_module(class_reference, pretrained_model_name_or_path, **load_kwargs) + else: + import transformers + from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES + + class_name = CONFIG_MAPPING_NAMES.get(getattr(config, "model_type", None) or "") + if class_name is not None: + config_cls = getattr(transformers, class_name, None) + if config_cls is None: + return config, False replacement = config_cls.from_pretrained(pretrained_model_name_or_path, **load_kwargs) original_quantization = getattr(config, "quantization_config", None) @@ -1121,6 +1175,8 @@ def _hf_source_load_kwargs( hf_model_cls: type, device: torch.device, hf_device_map_auto: bool, + hf_device_map_max_memory_gib: str | float | None = None, + hf_device_map_cpu_max_memory_gib: str | float | None = None, ) -> dict: """Build the HF-safe subset of recipe model kwargs for the source-load reference.""" hf_allowed_keys = { @@ -1157,6 +1213,13 @@ def _hf_source_load_kwargs( hf_kwargs["trust_remote_code"] = False if hf_device_map_auto: hf_kwargs["device_map"] = "auto" + # References too large for uncapped automatic placement (the 427B + # MiniMax-M3 fills every GPU and OOMs on the fused-expert concat + # transient) need the same GPU caps + CPU spill as the HF reload path. + max_memory = _hf_device_map_max_memory(hf_device_map_max_memory_gib, hf_device_map_cpu_max_memory_gib) + if max_memory is not None: + hf_kwargs["max_memory"] = max_memory + print(f"[Phase 0] Automatic device-map memory limits: {max_memory}") if ( "device_map" not in hf_kwargs and not hf_kwargs["trust_remote_code"] @@ -1251,6 +1314,15 @@ def _release_model_memory() -> None: torch.cuda.empty_cache() +# Leaf names distinctive enough to register as layout-independent fp32 +# aliases. Transformers matches _keep_in_fp32_modules_strict entries as +# unanchored substrings, so only leaves that cannot collide with unrelated +# modules qualify — a generic leaf such as ``proj`` or ``scale`` (from +# Gemma4's ``router.proj``/``router.scale``) would pin ``q_proj``, +# ``down_proj``, and friends fp32 across the whole bf16 reference. +_HF_FP32_LEAF_ALIASES = ("e_score_correction_bias",) + + def _hf_fp32_module_names(hf_config: object) -> tuple[str, ...]: """Infer vanilla-HF fp32 names from AutoModel's model-owned checkpoint contract.""" from nemo_automodel._transformers.model_init import _resolve_custom_model_cls_for_config @@ -1264,6 +1336,16 @@ def _hf_fp32_module_names(hf_config: object) -> tuple[str, ...]: for name in getattr(model_cls, "_keep_in_fp32_modules_strict", None) or (): if name not in module_names: module_names.append(name) + # AutoModel strict names use AutoModel module paths, but vanilla HF + # layouts can hang the same tensor off a different parent (in-tree + # MiniMax-M2 keeps e_score_correction_bias on ``mlp``, not + # ``mlp.gate``), so the AutoModel-path entry silently fails to match + # and the reference's router bias was cast to bf16 — scrambling 30-73% + # of knife-edge top-k selections per layer (AMINT-286). Also register + # the distinctive leaf so any layout keeps the tensor in fp32. + leaf = name.rsplit(".", 1)[-1] + if leaf in _HF_FP32_LEAF_ALIASES and leaf not in module_names: + module_names.append(leaf) return tuple(module_names) @@ -1527,6 +1609,8 @@ def _prepare_source_load_reference( trust_remote_code: bool | None, experts_implementation: str | None, hf_device_map_auto: bool, + hf_device_map_max_memory_gib: str | float | None = None, + hf_device_map_cpu_max_memory_gib: str | float | None = None, hf_source_post_load_dequantize: bool, parity_tolerance_profile: str = "standard", ) -> tuple[torch.Tensor, bool | None, bool | None] | None: @@ -1554,6 +1638,8 @@ def _prepare_source_load_reference( trust_remote_code=trust_remote_code, experts_implementation=experts_implementation, hf_device_map_auto=hf_device_map_auto, + hf_device_map_max_memory_gib=hf_device_map_max_memory_gib, + hf_device_map_cpu_max_memory_gib=hf_device_map_cpu_max_memory_gib, hf_source_post_load_dequantize=hf_source_post_load_dequantize, parity_tolerance_profile=parity_tolerance_profile, ) @@ -1575,6 +1661,8 @@ def _prepare_source_load_reference_rank0( trust_remote_code: bool | None, experts_implementation: str | None, hf_device_map_auto: bool, + hf_device_map_max_memory_gib: str | float | None = None, + hf_device_map_cpu_max_memory_gib: str | float | None = None, hf_source_post_load_dequantize: bool, parity_tolerance_profile: str = "standard", ) -> tuple[torch.Tensor, bool | None, bool | None]: @@ -1607,6 +1695,8 @@ def _prepare_source_load_reference_rank0( hf_model_cls=hf_model_cls, device=device, hf_device_map_auto=hf_device_map_auto, + hf_device_map_max_memory_gib=hf_device_map_max_memory_gib, + hf_device_map_cpu_max_memory_gib=hf_device_map_cpu_max_memory_gib, ) requested_attn_implementation = model_kwargs.get("attn_implementation") if hf_kwargs.get("attn_implementation") != requested_attn_implementation: @@ -1655,6 +1745,10 @@ def _prepare_source_load_reference_rank0( # Pass the faithful config explicitly so from_pretrained's internal # AutoConfig resolution cannot re-select the AutoModel-owned class. hf_kwargs["config"] = hf_config + if _repair_legacy_partial_rotary_config(hf_config): + # The repaired spec only reaches the model when the config object is + # passed explicitly; from_pretrained otherwise re-reads config.json. + hf_kwargs["config"] = hf_config model_load_context = _hf_model_load_context( trust_remote_code=trust_remote_code, @@ -2185,6 +2279,12 @@ def _run_vanilla_hf_reload( # Pass the faithful config explicitly so from_pretrained's internal # AutoConfig resolution cannot re-select the AutoModel-owned class. hf_kwargs["config"] = hf_config + if _repair_legacy_partial_rotary_config(hf_config): + # The repaired spec only reaches the model when the config object + # is passed explicitly; from_pretrained otherwise re-reads + # config.json (the consolidated export copies the source config, + # so it carries the same legacy rotary_dim field). + hf_kwargs["config"] = hf_config # Load the reference model straight onto the target GPU. Materialising a # 14B checkpoint on CPU and then ``.to(device)`` costs ~50-225s, and that # rank-0-only stall trips the NCCL watchdog while the other ranks idle at @@ -2432,6 +2532,8 @@ def _run_process_isolated_checkpoint_phase( trust_remote_code=custom_args.get("trust_remote_code"), experts_implementation=custom_args.get("experts_implementation", None), hf_device_map_auto=bool(custom_args.get("hf_device_map_auto", False)), + hf_device_map_max_memory_gib=custom_args.get("hf_device_map_max_memory_gib"), + hf_device_map_cpu_max_memory_gib=custom_args.get("hf_device_map_cpu_max_memory_gib"), hf_source_post_load_dequantize=bool(custom_args.get("hf_source_post_load_dequantize", False)), parity_tolerance_profile=_comparison_profile(custom_args, "source_load"), ) @@ -2993,6 +3095,8 @@ def run_checkpoint_robustness( trust_remote_code=trust_remote_code, experts_implementation=experts_implementation, hf_device_map_auto=hf_device_map_auto, + hf_device_map_max_memory_gib=custom_args.get("hf_device_map_max_memory_gib"), + hf_device_map_cpu_max_memory_gib=custom_args.get("hf_device_map_cpu_max_memory_gib"), hf_source_post_load_dequantize=hf_source_post_load_dequantize, parity_tolerance_profile=_comparison_profile(custom_args, "source_load"), ) diff --git a/tests/unit_tests/ci_tests/test_checkpoint_robustness_hf_kwargs.py b/tests/unit_tests/ci_tests/test_checkpoint_robustness_hf_kwargs.py index f89d988f05..5a5ef1836a 100644 --- a/tests/unit_tests/ci_tests/test_checkpoint_robustness_hf_kwargs.py +++ b/tests/unit_tests/ci_tests/test_checkpoint_robustness_hf_kwargs.py @@ -14,6 +14,7 @@ import json from contextlib import nullcontext +from copy import deepcopy from types import SimpleNamespace from unittest.mock import Mock, patch @@ -56,6 +57,7 @@ _prepare_consolidated_hf_cache_once, _raise_distributed_failure, _record_deferred_failure, + _repair_legacy_partial_rotary_config, _repeatability_policy, _replace_nemo_owned_reference_config, _resolve_hf_attn_implementation, @@ -1085,6 +1087,8 @@ def test_process_isolated_source_load_reference_persists_hf_artifacts(tmp_path): trust_remote_code=True, experts_implementation=None, hf_device_map_auto=True, + hf_device_map_max_memory_gib=None, + hf_device_map_cpu_max_memory_gib=None, hf_source_post_load_dequantize=False, parity_tolerance_profile="standard", ) @@ -1294,7 +1298,13 @@ class TinyAutoModel: "nemo_automodel._transformers.model_init._resolve_custom_model_cls_for_config", return_value=TinyAutoModel, ): - assert _hf_fp32_module_names(hf_config) == ("rotary_emb", "router.e_score_correction_bias") + # Dotted strict names also register their distinctive leaf so vanilla + # layouts with a different parent path stay covered. + assert _hf_fp32_module_names(hf_config) == ( + "rotary_emb", + "router.e_score_correction_bias", + "e_score_correction_bias", + ) def test_hf_fp32_module_names_combines_gdn_and_generic_contracts_without_duplicates(): @@ -1716,6 +1726,70 @@ def test_record_deferred_failure_preserves_all_comparison_failures(): assert failures == ["Phase 4 HF reload parity:\nHF parity failed"] +def _legacy_partial_rotary_minimax_config(**overrides): + """Tiny in-tree MiniMax-M2 config built from checkpoint-style legacy fields.""" + from transformers import AutoConfig + + kwargs = dict( + vocab_size=128, + hidden_size=64, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=32, + rotary_dim=16, + num_local_experts=4, + num_experts_per_tok=2, + max_position_embeddings=128, + ) + kwargs.update(overrides) + return AutoConfig.for_model("minimax_m2", **kwargs) + + +def test_repair_legacy_partial_rotary_derives_factor_from_rotary_dim(): + config = _legacy_partial_rotary_minimax_config() + + factor_missing_before = not config.rope_parameters.get("partial_rotary_factor") + assert _repair_legacy_partial_rotary_config(config) is factor_missing_before + assert config.rope_parameters["partial_rotary_factor"] == pytest.approx(0.5) + # A second pass finds the factor present and must not report a repair. + assert _repair_legacy_partial_rotary_config(config) is False + + +def test_repaired_minimax_m2_config_rotates_only_rotary_dim(): + from transformers import AutoModelForCausalLM as HFAutoModelForCausalLM + + config = _legacy_partial_rotary_minimax_config() + _repair_legacy_partial_rotary_config(config) + + model = HFAutoModelForCausalLM.from_config(config) + # inv_freq carries one frequency per rotated dim pair: rotary_dim // 2, + # not head_dim // 2 (the full-rotation failure mode from AMINT-286). + assert model.model.rotary_emb.inv_freq.shape[0] == config.rotary_dim // 2 + + +def test_repair_legacy_partial_rotary_is_noop_without_legacy_spec(): + from transformers import AutoConfig + + llama = AutoConfig.for_model( + "llama", + vocab_size=128, + hidden_size=64, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + ) + rope_before = deepcopy(getattr(llama, "rope_parameters", None)) + assert _repair_legacy_partial_rotary_config(llama) is False + assert getattr(llama, "rope_parameters", None) == rope_before + + full_rotary = _legacy_partial_rotary_minimax_config(rotary_dim=32) + rope_before = deepcopy(full_rotary.rope_parameters) + assert _repair_legacy_partial_rotary_config(full_rotary) is False + assert full_rotary.rope_parameters == rope_before + class _FakeNemoOwnedConfig(PretrainedConfig): """Stands in for an AutoModel component config registered into AutoConfig.""" @@ -1769,6 +1843,32 @@ def test_replace_nemo_owned_reference_config_preserves_dequantize_request(tmp_pa assert replaced.quantization_config["dequantize"] is True +class _FakeNemoOwnedLlamaConfig(PretrainedConfig): + """AutoModel-owned stand-in whose model_type shadows an in-tree class.""" + + model_type = "llama" + + +_FakeNemoOwnedLlamaConfig.__module__ = "nemo_automodel.components.models.test_only.config" + + +def test_replace_nemo_owned_reference_config_resolves_in_tree_class(tmp_path): + """Registered configs shadowing an in-tree model_type resolve to the in-tree + class for built-in references (the minimax_m3_vl vision-tower crash).""" + from transformers import LlamaConfig + + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "vocab_size": 64, "hidden_size": 32, "num_hidden_layers": 1}) + ) + hijacked = _FakeNemoOwnedLlamaConfig() + + replaced, did_replace = _replace_nemo_owned_reference_config(hijacked, tmp_path, trust_remote_code=False) + + assert did_replace is True + assert type(replaced) is LlamaConfig + assert replaced.hidden_size == 32 + + def test_replace_nemo_owned_reference_config_noop_cases(tmp_path): from transformers import AutoConfig @@ -1801,6 +1901,33 @@ def test_hf_source_load_kwargs_drops_nemo_owned_recipe_config(): assert "config" not in hf_kwargs +def test_hf_source_load_kwargs_applies_device_map_memory_caps(): + """Phase 0 must honor the same GPU/CPU caps as the HF reload path: the 427B + MiniMax-M3 reference OOMs under uncapped device_map=auto placement.""" + with ( + patch( + "transformers.PretrainedConfig.get_config_dict", + return_value=({"model_type": "unknown_remote_model"}, {}), + ), + patch("torch.cuda.device_count", return_value=2), + ): + hf_kwargs = _hf_source_load_kwargs( + {"attn_implementation": "eager"}, + pretrained_model_name_or_path="model-path", + source_dtype=torch.bfloat16, + trust_remote_code=True, + experts_implementation=None, + hf_model_cls=AutoModelForCausalLM, + device=torch.device("cpu"), + hf_device_map_auto=True, + hf_device_map_max_memory_gib=55, + hf_device_map_cpu_max_memory_gib=512, + ) + + assert hf_kwargs["device_map"] == "auto" + assert hf_kwargs["max_memory"] == {0: "55GiB", 1: "55GiB", "cpu": "512GiB"} + + def test_hf_source_load_kwargs_keeps_hf_recipe_config(): from transformers import AutoConfig @@ -1889,3 +2016,39 @@ def fused_kda_gate(g, A, head_k_dim, g_bias=None, beta=1.0, threshold=20.0): assert gate.fused_kda_gate is fused_kda_gate assert kda.fused_kda_gate is fused_kda_gate + + +def test_hf_fp32_module_names_cover_vanilla_layout_differences(): + """The fp32 contract must reach tensors whose vanilla-HF parent path differs. + + AutoModel's strict name is ``mlp.gate.e_score_correction_bias``, but in-tree + MiniMax-M2 keeps the buffer at ``mlp.e_score_correction_bias``; without the + leaf entry the HF reference silently casts the router bias to bf16. + """ + + class TinyAutoModel: + _keep_in_fp32_modules_strict = [ + "mlp.gate.e_score_correction_bias", + "router.weight", + "norm.bias", + # Gemma4-style entries: their leaves are unanchored-substring traps + # ("proj" matches q_proj/down_proj/...) and must not become aliases. + "router.proj", + "router.scale", + ] + + hf_config = SimpleNamespace(architectures=["TinyForCausalLM"]) + with patch( + "nemo_automodel._transformers.model_init._resolve_custom_model_cls_for_config", + return_value=TinyAutoModel, + ): + names = _hf_fp32_module_names(hf_config) + + assert "mlp.gate.e_score_correction_bias" in names + assert "e_score_correction_bias" in names + # The full AutoModel paths always pass through unchanged. + assert "router.proj" in names and "router.scale" in names + # Generic leaves would pin unrelated modules fp32 (Transformers matches + # these names as unanchored substrings) and must not be added. + for generic_leaf in ("weight", "bias", "proj", "scale"): + assert generic_leaf not in names diff --git a/tests/unit_tests/models/minimax_m2/__init__.py b/tests/unit_tests/models/minimax_m2/__init__.py index 26496bfed7..c03552b02d 100644 --- a/tests/unit_tests/models/minimax_m2/__init__.py +++ b/tests/unit_tests/models/minimax_m2/__init__.py @@ -1 +1,13 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit_tests/models/minimax_m2/test_minimax_m2_router_precision.py b/tests/unit_tests/models/minimax_m2/test_minimax_m2_router_precision.py new file mode 100644 index 0000000000..b9159b7d9f --- /dev/null +++ b/tests/unit_tests/models/minimax_m2/test_minimax_m2_router_precision.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MiniMax-M2 router precision contract. + +Released MiniMax-M2 checkpoints store the router gate weight in fp32 and the +HF reference projects with ``hidden_states.to(weight.dtype)``, so the +checkpoint-faithful router is fp32 end to end: fp32 parameter, fp32 +projection, fp32 scoring, fp32 selected weights (AMINT-286). +""" + +import torch +from transformers import AutoConfig + +from nemo_automodel.components.models.common import BackendConfig +from nemo_automodel.components.models.minimax_m2.model import MiniMaxM2ForCausalLM + +TINY = dict( + vocab_size=128, + hidden_size=64, + intermediate_size=32, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=32, + rotary_dim=16, + num_local_experts=8, + num_experts_per_tok=2, + max_position_embeddings=128, +) + + +def _cpu_backend() -> BackendConfig: + return BackendConfig( + linear="torch", + attn="sdpa", + rms_norm="torch", + rope_fusion=False, + dispatcher="torch", + experts="torch", + fake_balanced_gate=False, + enable_hf_state_dict_adapter=False, + ) + + +def test_router_fp32_contract_is_model_owned(): + config = AutoConfig.for_model("minimax_m2", torch_dtype="bfloat16", **TINY) + model = MiniMaxM2ForCausalLM(config, backend=_cpu_backend()).eval() + + assert model.model.backend.gate_precision == torch.float32 + assert model.model.moe_config.router_weights_fp32 is True + assert "mlp.gate.weight" in MiniMaxM2ForCausalLM._keep_in_fp32_modules_strict + assert "mlp.gate.e_score_correction_bias" in MiniMaxM2ForCausalLM._keep_in_fp32_modules_strict + + # After the model-wide bf16 cast, the fp32 contract keeps the router gate + # parameter and bias in fp32 while the rest of the model is bf16. + model.initialize_weights(buffer_device=torch.device("cpu"), dtype=torch.bfloat16) + gate = model.model.layers["0"].mlp.gate + assert gate.weight.dtype == torch.float32 + assert gate.e_score_correction_bias.dtype == torch.float32 + assert model.model.layers["0"].self_attn.q_proj.weight.dtype == torch.bfloat16 + + # Selected routing weights stay fp32 through the gate output, matching the + # HF reference's top_k_weights.to(router_logits.dtype) with an fp32 weight. + # x: Tensor of shape [tokens, hidden] in bf16, like real routed inputs. + x = torch.randn(8, TINY["hidden_size"], dtype=torch.bfloat16) + weights, indices, _aux = gate(x, torch.ones(8, dtype=torch.bool), None) + assert weights.dtype == torch.float32 + assert indices.shape == (8, TINY["num_experts_per_tok"]) + + +def test_gate_is_fp32_at_construction_for_fsdp_dtype_grouping(): + """The gate must be fp32 from allocation, before any init or checkpoint cast. + + FSDP shards the freshly constructed (meta/from_pretrained) module: a + bf16-allocated gate weight with an fp32-pinned compute dtype shares its + module with the fp32 correction-bias buffer, which FSDP cannot isolate + (pipeline 64344786: "FSDP could not isolate parameters with a distinct + dtype from siblings in the same module: mlp.gate.weight"). + """ + import torch.distributed.fsdp as fsdp + + from nemo_automodel.components.distributed.parallelizer_utils import fully_shard_by_dtype + + config = AutoConfig.for_model("minimax_m2", torch_dtype="bfloat16", **TINY) + model = MiniMaxM2ForCausalLM(config, backend=_cpu_backend()) + block = model.model.layers["0"] + + # No initialize_weights on purpose: this is the state FSDP shards. + assert block.mlp.gate.weight.dtype == torch.float32 + assert block.mlp.gate.e_score_correction_bias.dtype == torch.float32 + + fully_shard_by_dtype( + block, + mesh=None, + mp_policy=fsdp.MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32), + offload_policy=None, + fp32_compute_module_names=tuple(MiniMaxM2ForCausalLM._keep_in_fp32_modules_strict), + fully_shard_fn=lambda *args, **kwargs: None, + ) + + +def test_explicit_gate_precision_override_is_preserved(): + config = AutoConfig.for_model("minimax_m2", torch_dtype="bfloat16", **TINY) + backend = _cpu_backend() + backend.gate_precision = torch.bfloat16 + model = MiniMaxM2ForCausalLM(config, backend=backend).eval() + assert model.model.backend.gate_precision == torch.bfloat16 diff --git a/tests/unit_tests/models/minimax_m3_vl/test_minimax_m3_router_precision.py b/tests/unit_tests/models/minimax_m3_vl/test_minimax_m3_router_precision.py new file mode 100644 index 0000000000..bbff549257 --- /dev/null +++ b/tests/unit_tests/models/minimax_m3_vl/test_minimax_m3_router_precision.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MiniMax-M3 router precision contract. + +Released MiniMax-M3 checkpoints store the router gate weight in fp32 and the +correction bias as the same 1e-3-quantized fp32 lattice as MiniMax-M2.7, so +the checkpoint-faithful router keeps both tensors fp32 from allocation +through load (AMINT-286 pattern). +""" + +import torch + +from nemo_automodel.components.models.common import BackendConfig +from nemo_automodel.components.models.minimax_m3_vl.config import MiniMaxM3VLTextConfig +from nemo_automodel.components.models.minimax_m3_vl.model import ( + MiniMaxM3SparseForCausalLM, + MiniMaxM3SparseForConditionalGeneration, +) +from tests.unit_tests.models.minimax_m3_vl.conftest import TINY_CFG + + +def _first_moe_block(model): + """First decoder block with a routed MoE mlp (M3 leads with dense layers).""" + for block in model.model.layers.values(): + if hasattr(block.mlp, "gate"): + return block + raise AssertionError("tiny config produced no MoE layer") + + +def _cpu_backend() -> BackendConfig: + return BackendConfig( + linear="torch", + attn="sdpa", + rms_norm="torch", + rope_fusion=False, + dispatcher="torch", + experts="torch", + fake_balanced_gate=False, + enable_hf_state_dict_adapter=False, + ) + + +def test_router_fp32_contract_is_model_owned(): + config = MiniMaxM3VLTextConfig(torch_dtype="bfloat16", **TINY_CFG) + model = MiniMaxM3SparseForCausalLM(config, backend=_cpu_backend()).eval() + + assert model.model.backend.gate_precision == torch.float32 + for cls in (MiniMaxM3SparseForCausalLM, MiniMaxM3SparseForConditionalGeneration): + assert "mlp.gate.weight" in cls._keep_in_fp32_modules_strict + assert "mlp.gate.e_score_correction_bias" in cls._keep_in_fp32_modules_strict + + # After the model-wide bf16 cast, the fp32 contract keeps the router gate + # parameter and bias fp32 while the rest of the model is bf16. + model.initialize_weights(dtype=torch.bfloat16) + moe_block = _first_moe_block(model) + gate = moe_block.mlp.gate + assert gate.weight.dtype == torch.float32 + assert gate.e_score_correction_bias.dtype == torch.float32 + assert moe_block.self_attn.q_proj.weight.dtype == torch.bfloat16 + + +def test_gate_is_fp32_at_construction_for_fsdp_dtype_grouping(): + """The gate must be fp32 from allocation, before any init or checkpoint cast. + + FSDP shards the freshly constructed module: a bf16-allocated gate weight + whose compute dtype is pinned fp32 shares its module with the fp32 + correction-bias buffer, which FSDP cannot isolate (the MiniMax-M2.7 + failure from pipeline 64344786). + """ + import torch.distributed.fsdp as fsdp + + from nemo_automodel.components.distributed.parallelizer_utils import fully_shard_by_dtype + + config = MiniMaxM3VLTextConfig(torch_dtype="bfloat16", **TINY_CFG) + model = MiniMaxM3SparseForCausalLM(config, backend=_cpu_backend()) + block = _first_moe_block(model) + + # No initialize_weights on purpose: this is the state FSDP shards. + assert block.mlp.gate.weight.dtype == torch.float32 + assert block.mlp.gate.e_score_correction_bias.dtype == torch.float32 + + fully_shard_by_dtype( + block, + mesh=None, + mp_policy=fsdp.MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32), + offload_policy=None, + fp32_compute_module_names=tuple(MiniMaxM3SparseForCausalLM._keep_in_fp32_modules_strict), + fully_shard_fn=lambda *args, **kwargs: None, + ) diff --git a/tests/unit_tests/moe/test_layers.py b/tests/unit_tests/moe/test_layers.py index 759ffc01b2..cd17b6e6f6 100644 --- a/tests/unit_tests/moe/test_layers.py +++ b/tests/unit_tests/moe/test_layers.py @@ -1876,3 +1876,67 @@ def test_apply_bias_is_not_compiled(self): # torch.compile wraps functions in OptimizedModule or similar assert not hasattr(_apply_bias, "_torchdynamo_orig_callable"), "_apply_bias should not be torch.compiled" + + +class TestSigmoidGateScoringPrecision: + """Sigmoid routing must score in fp32 by default, like the softmax path. + + HF sigmoid-router references compute ``sigmoid(logits.float())``; scoring in + bf16 quantizes scores at ~2e-3, which flips knife-edge selections against + fine-grained ``e_score_correction_bias`` lattices (AMINT-286). + """ + + def _sigmoid_config(self): + return MoEConfig( + n_routed_experts=16, + n_shared_experts=0, + n_activated_experts=4, + n_expert_groups=0, + n_limited_groups=0, + train_gate=True, + gate_bias_update_factor=0.0, + aux_loss_coeff=0.0, + score_func="sigmoid", + route_scale=1.0, + dim=64, + inter_dim=128, + moe_inter_dim=128, + norm_topk_prob=True, + router_bias=False, + expert_bias=False, + expert_activation="swiglu", + force_e_score_correction_bias=True, + dtype=torch.bfloat16, + ) + + def test_sigmoid_scoring_matches_fp32_reference_on_bf16_inputs(self): + torch.manual_seed(0) + config = self._sigmoid_config() + gate = Gate(config) + with torch.no_grad(): + gate.weight.copy_(torch.randn_like(gate.weight) * 0.05) + # Knife-edge lattice bias like the MiniMax-M2.7 / GLM-4.7 checkpoints: + # large magnitude, 1e-3 spacing. + gate.e_score_correction_bias.copy_(8.0 + torch.arange(config.n_routed_experts, dtype=torch.float32) * 1e-3) + gate.eval() + + # x: Tensor of shape [tokens, hidden] in bf16, like real routed inputs. + x = torch.randn(512, config.dim, dtype=torch.bfloat16) + token_mask = torch.ones(x.shape[0], dtype=torch.bool) + weights, indices, _aux = gate(x, token_mask, None) + + # fp32 reference: the bf16 gate matmul followed by fp32 sigmoid, + # bias-augmented selection, and fp32 top-k normalization. + scores_bf16 = F.linear(x, gate.weight.to(x.dtype)) + ref_scores = torch.sigmoid(scores_bf16.float()) + choice = ref_scores + gate.e_score_correction_bias + ref_indices = torch.topk(choice, config.n_activated_experts, dim=-1)[1] + + assert torch.equal(indices.sort(dim=-1).values, ref_indices.sort(dim=-1).values) + + ref_weights = ref_scores.gather(1, indices) + ref_weights = ref_weights / (ref_weights.sum(dim=-1, keepdim=True) + 1e-20) + # The gate casts final weights back to the input dtype, mirroring the + # HF reference's top_k_weights.to(router_logits.dtype). + assert weights.dtype == x.dtype + assert torch.equal(weights, ref_weights.to(x.dtype))