Skip to content

vllm.models.glm5next.common.model

Classes:

Functions:

_LOGIT_SCALE = 1.0 module-attribute

Output logit scale. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_MHC_POST_MULT_VALUE = 2.0 module-attribute

mHC post-multiplier. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_MHC_TAU = 0.05 module-attribute

mHC routing temperature. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_VISION_RMS_NORM_EPS = 1e-06 module-attribute

Vision tower RMSNorm epsilon.

GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5, but the vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and produces repetitive/degraded image descriptions, so force the trained value regardless of the checkpoint field.

Glm5NextDecoderLayer

Bases: Module

Methods:

  • split_hc_fns –

    Make, or rewrite in place, the BF16 pieces of hc_attn_fn and hc_ffn_fn.

Source code in vllm/models/glm5next/common/model.py
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
class Glm5NextDecoderLayer(nn.Module):
    def __init__(
        self,
        vllm_config: VllmConfig,
        config: Glm5NextTextConfig,
        layer_idx: int,
        prefix: str = "",
        topk_indices_buffer: torch.Tensor | None = None,
        is_mtp_layer: bool = False,
        **kwargs,
    ) -> None:
        super().__init__()

        cache_config = vllm_config.cache_config
        quant_config = vllm_config.quant_config
        parallel_config = vllm_config.parallel_config

        self.hidden_size = config.hidden_size
        self.layer_idx = layer_idx
        self.is_moe = _is_moe(config)
        self.num_hidden_layers = config.num_hidden_layers
        self.rms_norm_eps = config.rms_norm_eps
        self.num_experts = config.n_routed_experts
        self.is_mtp_layer = is_mtp_layer
        self.mhc = config.mhc
        is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx)
        self.layer_kind = "kda" if is_kda_layer else "mla"
        self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe

        if is_kda_layer:
            self.self_attn = Glm5NextLinearAttention(
                config=config,
                vllm_config=vllm_config,
                prefix=f"{prefix}.self_attn",
            )
        else:
            # MLA layers require the latent head dims, which are guaranteed set
            # on MLA configs; narrow away the `int | None`.
            assert config.v_head_dim is not None
            assert config.kv_lora_rank is not None
            self.self_attn = Glm5NextMLAAttention(
                vllm_config=vllm_config,
                config=config,
                hidden_size=self.hidden_size,
                num_heads=config.num_attention_heads,
                qk_nope_head_dim=config.qk_nope_head_dim,
                qk_rope_head_dim=config.qk_rope_head_dim,
                v_head_dim=config.v_head_dim,
                q_lora_rank=config.q_lora_rank,
                kv_lora_rank=config.kv_lora_rank,
                max_position_embeddings=config.max_position_embeddings,
                cache_config=cache_config,
                quant_config=None,  # MLA projections run in BF16 (Glm5NextMLAAttention)
                prefix=f"{prefix}.self_attn",
                topk_indices_buffer=topk_indices_buffer,
                skip_rope=config.mla_use_nope,
                # MTP draft steps reuse each token slot's top-k, which
                # ShardQ's compact per-rank layout would break.
                shardq_eligible=not is_mtp_layer,
            )

        # MTP layers sit past the base model's hidden layers (layer_idx >=
        # num_hidden_layers), so they're outside mlp_layer_types; default them
        # to the last base layer's MLP type (sparse/MoE for these checkpoints).
        mlp_layer_types = config.mlp_layer_types
        mlp_type = (
            mlp_layer_types[layer_idx]
            if layer_idx < len(mlp_layer_types)
            else (mlp_layer_types[-1] if mlp_layer_types else "sparse")
        )
        if self.is_moe and self.num_experts is not None and mlp_type == "sparse":
            self.mlp = Glm5NextMoE(
                config=config,
                parallel_config=parallel_config,
                quant_config=quant_config,
                prefix=f"{prefix}.mlp",
            )
        else:
            self.mlp = Glm5NextMLP(
                hidden_size=self.hidden_size,
                intermediate_size=config.intermediate_size,
                hidden_act=config.hidden_act,
                quant_config=quant_config,
                is_sequence_parallel=self.is_sequence_parallel,
                prefix=f"{prefix}.mlp",
                swiglu_limit=config.swiglu_limit,
            )
        self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        # Cached for the hot forward path (isinstance per layer per step).
        self._mlp_is_moe = isinstance(self.mlp, Glm5NextMoE)
        # In SP, the attention output projection leaves a partial sum; the
        # decoder-layer reduce_scatter after attention completes it (DSv4 pattern).
        # MTP layers use the non-mHC path which has no sp_reduce_scatter, so
        # their o_proj must still reduce normally.
        if self.is_sequence_parallel and not is_mtp_layer:
            self.self_attn.o_proj.reduce_results = False
        self.post_attention_layernorm = RMSNorm(
            config.hidden_size, eps=config.rms_norm_eps
        )

        # The BF16 pieces of hc_*_fn when FlashInfer's fused kernel serves this
        # GPU; made after loading, refreshed in place.
        self.use_mega_mhc = False
        self.mega_mhc_min_tokens = 0
        self.hc_attn_fn_split: torch.Tensor | None = None
        self.hc_ffn_fn_split: torch.Tensor | None = None
        if self.mhc and not is_mtp_layer:
            # mhc config
            self.mhc_num_residual_streams = config.hc_mult
            self.mhc_tau = _MHC_TAU
            self.hc_eps = config.hc_eps
            self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters
            self.mhc_post_mult_value = _MHC_POST_MULT_VALUE

            n = config.hc_mult
            d_model = n * self.hidden_size
            mix_hc = (2 + n) * n

            self.n = n

            # attn hc
            self.hc_attn_fn = nn.Parameter(
                torch.empty(mix_hc, d_model, dtype=torch.float32)
            )
            self.hc_attn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32))
            self.hc_attn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32))

            # ffn hc
            self.hc_ffn_fn = nn.Parameter(
                torch.empty(mix_hc, d_model, dtype=torch.float32)
            )
            self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32))
            self.hc_ffn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32))

            capability = current_platform.get_device_capability()
            min_tokens = _MEGA_MHC_MIN_TOKENS.get(
                -1 if capability is None else capability.to_int()
            )
            if min_tokens is not None and mega_mhc_supported(self.hidden_size, n):
                self.use_mega_mhc = True
                self.mega_mhc_min_tokens = min_tokens
                logger.info_once(
                    "GLM-5.3 mHC uses FlashInfer mega mHC from %d tokens per step.",
                    self.mega_mhc_min_tokens,
                )
                # Weight updates rewrite hc_*_fn in place through these loaders,
                # which rewrite the BF16 pieces in place too.
                self.hc_attn_fn.weight_loader = self._load_hc_attn_fn
                self.hc_ffn_fn.weight_loader = self._load_hc_ffn_fn
                if vllm_config.kernel_config.enable_jit_warmup:
                    MEGA_MHC_KERNEL.register_warmup(
                        max_num_batched_tokens=(
                            vllm_config.scheduler_config.max_num_batched_tokens
                        )
                    )

            self.mhc_pre_op = MHCPreOp()
            self.mhc_post_op = MHCPostOp()
            self.mhc_fused_post_pre_op = MHCFusedPostPreOp()

            if vllm_config.kernel_config.enable_jit_warmup:
                from vllm.model_executor.kernels.mhc.tilelang_kernels import (
                    _HC_PRENORM_GEMM_TILELANG_KERNEL,
                    _MHC_FUSED_TILELANG_KERNEL,
                    _MHC_POST_TILELANG_KERNEL,
                    _MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
                )
                from vllm.utils.deep_gemm import is_deep_gemm_supported

                include_pre_gemm_splits = is_deep_gemm_supported()
                _MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup(
                    vllm_config,
                    hidden_size=self.hidden_size,
                    hc_mult=self.n,
                    use_norm_weight=True,
                    include_pre_gemm_splits=include_pre_gemm_splits,
                    include_broadcast_splits=False,
                    rms_eps=self.rms_norm_eps,
                    hc_pre_eps=self.hc_eps,
                    hc_sinkhorn_eps=self.hc_eps,
                    hc_post_mult_value=self.mhc_post_mult_value,
                    sinkhorn_repeat=self.mhc_sinkhorn_iterations,
                    norm_eps=(
                        self.input_layernorm.variance_epsilon,
                        self.post_attention_layernorm.variance_epsilon,
                    ),
                )
                if not include_pre_gemm_splits:
                    _HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup(
                        vllm_config,
                        hidden_size=self.hidden_size,
                        hc_mult=self.n,
                        n_out=self.n * (2 + self.n),
                    )
                _MHC_POST_TILELANG_KERNEL.register_warmup(
                    hidden_size=self.hidden_size,
                    hc_mult=self.n,
                )
                _MHC_FUSED_TILELANG_KERNEL.register_warmup(
                    vllm_config,
                    hidden_size=self.hidden_size,
                    hc_mult=self.n,
                )

    def _load_hc_attn_fn(self, param: torch.Tensor, loaded_weight: torch.Tensor):
        default_weight_loader(param, loaded_weight)
        # Layerwise reload also runs loaders on meta tensors, to count them.
        if self.hc_attn_fn_split is not None and not param.is_meta:
            split_mhc_fn(param, out=self.hc_attn_fn_split)

    def _load_hc_ffn_fn(self, param: torch.Tensor, loaded_weight: torch.Tensor):
        default_weight_loader(param, loaded_weight)
        if self.hc_ffn_fn_split is not None and not param.is_meta:
            split_mhc_fn(param, out=self.hc_ffn_fn_split)

    def split_hc_fns(self) -> None:
        """Make, or rewrite in place, the BF16 pieces of hc_attn_fn and hc_ffn_fn."""
        if not self.use_mega_mhc:
            return
        self.hc_attn_fn_split = split_mhc_fn(self.hc_attn_fn, self.hc_attn_fn_split)
        self.hc_ffn_fn_split = split_mhc_fn(self.hc_ffn_fn, self.hc_ffn_fn_split)

    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
        residual: torch.Tensor | None = None,
        post: torch.Tensor | None = None,
        comb: torch.Tensor | None = None,
    ) -> tuple[
        torch.Tensor,
        torch.Tensor | None,
        torch.Tensor | None,
        torch.Tensor | None,
    ]:
        # 70B or MTP layers: KDA + MoE without HC.
        if not self.mhc or self.is_mtp_layer:
            residual = hidden_states
            hidden_states = self.input_layernorm(hidden_states)

            attn_output = self.self_attn(
                hidden_states=hidden_states,
                positions=positions,
            )
            hidden_states, residual = self.post_attention_layernorm(
                attn_output, residual=residual
            )
            hidden_states = self.mlp(hidden_states)
            if self.is_mtp_layer:
                # Return the unsummed pair: the MTP caller feeds it straight
                # into shared_head's fused_add_rms_norm (one kernel instead of
                # a separate residual-add + norm). The sum itself is unchanged
                # (fp32-accumulated inside the fused kernel).
                return hidden_states, residual, None, None
            hidden_states = residual + hidden_states
            return hidden_states, residual, None, None

        # mHC start. `post`/`comb` carry the previous layer's deferred
        # hc_post inputs (its ffn-pre outputs); when present, fuse that
        # hc_post with this layer's attn hc_pre into one kernel (inter-layer
        # fusion). Layer 0 has no incoming state -> standalone hc_pre.
        x = hidden_states
        if post is None:
            if self.layer_idx == 0:
                x = hc_expand(x, self.n)
            residual = x
            post, comb, x = self.hc_pre(
                x,
                self.hc_attn_fn,
                self.hc_attn_scale,
                self.hc_attn_base,
                norm_weight=self.input_layernorm.weight.data,
                norm_eps=self.input_layernorm.variance_epsilon,
            )
        else:
            residual, post, comb, x = self.hc_fused_post_pre(
                x,
                residual,
                post,
                comb,
                self.hc_attn_fn,
                self.hc_attn_scale,
                self.hc_attn_base,
                norm_weight=self.input_layernorm.weight.data,
                norm_eps=self.input_layernorm.variance_epsilon,
                fn_split=self.hc_attn_fn_split,
            )

        # Attention needs the full token sequence; mHC above ran on the SP
        # shard. Gather for attention, scatter back afterward (DSv4 pattern).
        if self.is_sequence_parallel:
            x = sp_all_gather(x)[: positions.shape[0]]

        x = self.self_attn(
            hidden_states=x,
            positions=positions,
        )

        if self.is_sequence_parallel:
            x = sp_reduce_scatter(x)

        # Fuse post-attn hc_post + pre-FFN hc_pre (+ RMSNorm) into one kernel.
        residual, post, comb, x = self.hc_fused_post_pre(
            x,
            residual,
            post,
            comb,
            self.hc_ffn_fn,
            self.hc_ffn_scale,
            self.hc_ffn_base,
            norm_weight=self.post_attention_layernorm.weight.data,
            norm_eps=self.post_attention_layernorm.variance_epsilon,
            fn_split=self.hc_ffn_fn_split,
        )

        # Fully Connected
        if self._mlp_is_moe:
            x = self.mlp(x, already_sequence_parallel=self.is_sequence_parallel)
        else:
            x = self.mlp(x)

        # mHC end. The last mHC layer materializes its final hc_post (nothing
        # to fuse with) then contracts; every other layer defers its hc_post to
        # the next layer's fused pre, returning the state.
        if self.layer_idx == self.num_hidden_layers - 1:
            x = self.hc_post(x, residual, post, comb)
            x = hc_contract(x, self.n)
            return x, None, None, None

        return x, residual, post, comb

    def hc_pre(
        self,
        x: torch.Tensor,
        hc_fn: torch.Tensor,
        hc_scale: torch.Tensor,
        hc_base: torch.Tensor,
        norm_weight: torch.Tensor | None = None,
        norm_eps: float = 0.0,
    ):
        post_mix, res_mix, layer_input = self.mhc_pre_op(
            residual=x,
            fn=hc_fn,
            hc_scale=hc_scale,
            hc_base=hc_base,
            rms_eps=self.rms_norm_eps,
            hc_pre_eps=self.hc_eps,
            hc_sinkhorn_eps=self.hc_eps,
            hc_post_mult_value=self.mhc_post_mult_value,
            sinkhorn_repeat=self.mhc_sinkhorn_iterations,
            norm_weight=norm_weight,
            norm_eps=norm_eps,
        )
        return post_mix, res_mix, layer_input

    def hc_post(
        self,
        x: torch.Tensor,
        residual: torch.Tensor,
        post: torch.Tensor,
        comb: torch.Tensor,
    ):
        return self.mhc_post_op(x, residual, post, comb)

    def hc_fused_post_pre(
        self,
        x: torch.Tensor,
        residual: torch.Tensor,
        post: torch.Tensor,
        comb: torch.Tensor,
        hc_fn: torch.Tensor,
        hc_scale: torch.Tensor,
        hc_base: torch.Tensor,
        norm_weight: torch.Tensor | None = None,
        norm_eps: float = 0.0,
        fn_split: torch.Tensor | None = None,
    ):
        # The model runs eagerly under CUDA graphs, so the token count is a
        # concrete int and each captured size keeps its branch.
        if self.use_mega_mhc and residual.shape[0] >= self.mega_mhc_min_tokens:
            assert fn_split is not None and norm_weight is not None
            return torch.ops.vllm.mhc_fused_post_pre_flashinfer_mega(
                x,
                residual,
                post,
                comb,
                hc_fn,
                fn_split,
                hc_scale,
                hc_base,
                self.rms_norm_eps,
                self.hc_eps,
                self.hc_eps,
                self.mhc_post_mult_value,
                self.mhc_sinkhorn_iterations,
                norm_weight,
                norm_eps,
            )
        return self.mhc_fused_post_pre_op(
            x=x,
            residual=residual,
            post_layer_mix=post,
            comb_res_mix=comb,
            fn=hc_fn,
            hc_scale=hc_scale,
            hc_base=hc_base,
            rms_eps=self.rms_norm_eps,
            hc_pre_eps=self.hc_eps,
            hc_sinkhorn_eps=self.hc_eps,
            hc_post_mult_value=self.mhc_post_mult_value,
            sinkhorn_repeat=self.mhc_sinkhorn_iterations,
            n_splits=1,
            tile_n=1,
            norm_weight=norm_weight,
            norm_eps=norm_eps,
        )

split_hc_fns()

Make, or rewrite in place, the BF16 pieces of hc_attn_fn and hc_ffn_fn.

Source code in vllm/models/glm5next/common/model.py
def split_hc_fns(self) -> None:
    """Make, or rewrite in place, the BF16 pieces of hc_attn_fn and hc_ffn_fn."""
    if not self.use_mega_mhc:
        return
    self.hc_attn_fn_split = split_mhc_fn(self.hc_attn_fn, self.hc_attn_fn_split)
    self.hc_ffn_fn_split = split_mhc_fn(self.hc_ffn_fn, self.hc_ffn_fn_split)

Glm5NextMoE

Bases: Module

Source code in vllm/models/glm5next/common/model.py
class Glm5NextMoE(nn.Module):
    def __init__(
        self,
        config: Glm5NextTextConfig,
        parallel_config: ParallelConfig,
        quant_config: QuantizationConfig | None = None,
        prefix: str = "",
        routed_scale_mode: RoutedScaleMode = RoutedScaleMode.ROUTER,
    ):
        super().__init__()
        self.tp_size = get_tensor_model_parallel_world_size()
        self.tp_rank = get_tensor_model_parallel_rank()

        self.routed_scaling_factor = config.routed_scaling_factor

        self.ep_group = get_ep_group().device_group
        self.ep_rank = get_ep_group().rank_in_group
        self.ep_size = self.ep_group.size()
        self.n_routed_experts: int = config.n_routed_experts
        self.n_shared_experts: int = config.n_shared_experts

        self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe

        if config.hidden_act != "silu":
            raise ValueError(
                f"Unsupported activation: {config.hidden_act}. "
                "Only silu is supported for now."
            )

        self.router_dtype = _get_moe_router_dtype(config)
        self.gate = GateLinear(
            config.hidden_size,
            config.n_routed_experts,
            out_dtype=self.router_dtype,
            prefix=f"{prefix}.gate",
        )
        if config.topk_method == "noaux_tc":
            self.gate.e_score_correction_bias = nn.Parameter(
                torch.empty(config.n_routed_experts, dtype=torch.float32)
            )
        else:
            self.gate.e_score_correction_bias = None

        # Load balancing settings.
        eplb_config = parallel_config.eplb_config
        self.enable_eplb = parallel_config.enable_eplb

        self.n_redundant_experts = eplb_config.num_redundant_experts
        self.n_logical_experts = self.n_routed_experts
        self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
        self.n_local_physical_experts = self.n_physical_experts // self.ep_size

        self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
        self.physical_expert_end = (
            self.physical_expert_start + self.n_local_physical_experts
        )

        swiglu_limit = config.swiglu_limit
        vllm_config = get_current_vllm_config()
        self.use_mega_moe = (
            vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
        )
        self.is_fused_shared_expert_enabled = False
        if config.n_shared_experts is not None and not self.use_mega_moe:
            self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
                quant_config, prefix
            ) and _fused_shared_experts_tuned(parallel_config)
        # MegaMoE owns the shared-expert weights itself.
        if (
            config.n_shared_experts is None
            or self.is_fused_shared_expert_enabled
            or self.use_mega_moe
        ):
            self.shared_experts = None
        else:
            intermediate_size = config.moe_intermediate_size * config.n_shared_experts

            self.shared_experts = Glm5NextMLP(
                hidden_size=config.hidden_size,
                intermediate_size=intermediate_size,
                hidden_act=config.hidden_act,
                quant_config=quant_config,
                is_sequence_parallel=self.is_sequence_parallel,
                reduce_results=False,
                prefix=f"{prefix}.shared_experts",
                swiglu_limit=swiglu_limit,
            )

        if self.use_mega_moe:
            self._init_mega_moe_experts(vllm_config, config, quant_config, prefix)
            return

        self.experts = FusedMoEFactory(
            shared_experts=self.shared_experts,
            gate=self.gate,
            num_experts=config.n_routed_experts,
            top_k=config.num_experts_per_tok,
            hidden_size=config.hidden_size,
            intermediate_size=config.moe_intermediate_size,
            renormalize=config.norm_topk_prob,
            quant_config=quant_config,
            use_grouped_topk=True,
            num_expert_group=config.n_group,
            topk_group=config.topk_group,
            prefix=f"{prefix}.experts",
            scoring_func=config.scoring_func,
            routed_scaling_factor=self.routed_scaling_factor,
            routed_scale_mode=routed_scale_mode,
            e_score_correction_bias=self.gate.e_score_correction_bias,
            enable_eplb=self.enable_eplb,
            num_redundant_experts=self.n_redundant_experts,
            is_sequence_parallel=self.is_sequence_parallel,
            n_shared_experts=config.n_shared_experts
            if self.is_fused_shared_expert_enabled
            else None,
            fuse_shared_experts=self.is_fused_shared_expert_enabled,
            shared_expert_prefix=f"{prefix}.shared_experts",
            router_logits_dtype=self.gate.out_dtype,
            swiglu_limit=swiglu_limit,
        )

    def _init_mega_moe_experts(
        self,
        vllm_config: VllmConfig,
        config: Glm5NextTextConfig,
        quant_config: QuantizationConfig | None,
        prefix: str,
    ) -> None:
        """DeepGEMM MegaMoE, BF16xBF16 for an unquantized checkpoint and
        FP8xFP8 for a block-FP8 one: the kernel dispatches, runs the routed and
        shared experts and combines across the EP group."""
        from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
            MegaMoEActivation,
        )
        from vllm.models.glm5next.nvidia.mega_moe import Glm5NextMegaMoEExperts

        if not vllm_config.parallel_config.enable_expert_parallel:
            raise NotImplementedError(
                "GLM-5.3 MegaMoE requires expert parallel. Enable it with "
                "--enable-expert-parallel, or pick a different moe backend."
            )
        if self.tp_size != 1:
            raise NotImplementedError(
                "GLM-5.3 MegaMoE requires tensor_parallel_size=1 (DP + EP)."
            )
        if (config.n_group or 1) > 1 or (config.topk_group or 1) > 1:
            raise NotImplementedError("GLM-5.3 MegaMoE requires one expert group.")
        weight_block_size = getattr(quant_config, "weight_block_size", None)
        if quant_config is not None and weight_block_size is None:
            raise NotImplementedError(
                "GLM-5.3 MegaMoE requires a BF16 or a block-FP8 checkpoint "
                "(quantization_config.weight_block_size)."
            )
        if self.n_physical_experts % self.ep_size != 0:
            raise ValueError(
                f"n_physical_experts={self.n_physical_experts} must be divisible "
                f"by ep_size={self.ep_size}."
            )
        self.mega_renormalize = config.norm_topk_prob
        self.mega_scoring_func = config.scoring_func
        self.experts = Glm5NextMegaMoEExperts(
            vllm_config,
            num_experts=self.n_physical_experts,
            num_local_experts=self.n_local_physical_experts,
            experts_start_idx=self.physical_expert_start,
            num_logical_experts=self.n_logical_experts,
            top_k=config.num_experts_per_tok,
            hidden_size=config.hidden_size,
            intermediate_size=config.moe_intermediate_size,
            weight_block_size=None
            if weight_block_size is None
            else tuple(weight_block_size),
            num_shared_experts=config.n_shared_experts or 0,
            activation=MegaMoEActivation("swiglu", clamp=config.swiglu_limit),
            prefix=f"{prefix}.experts",
        )

    def finalize_mega_moe_weights(self) -> None:
        if self.use_mega_moe and self.experts.weight_block_size is not None:
            self.experts.requantize_block_fp8()

    def forward(
        self,
        hidden_states: torch.Tensor,
        already_sequence_parallel: bool = False,
    ) -> torch.Tensor:
        if self.use_mega_moe:
            router_logits, _ = self.gate(hidden_states)
            bias = self.gate.e_score_correction_bias
            topk_weights, topk_ids = fused_topk_bias(
                hidden_states=hidden_states,
                gating_output=router_logits,
                scoring_func=self.mega_scoring_func,
                e_score_correction_bias=bias.data if bias is not None else None,
                topk=self.experts.top_k,
                renormalize=self.mega_renormalize,
                indices_type=torch.int64,
                routed_scaling_factor=self.routed_scaling_factor,
            )
            return self.experts(hidden_states, topk_weights, topk_ids)

        num_tokens, hidden_dim = hidden_states.shape

        # Chunk the hidden states so they aren't replicated across TP ranks.
        # This avoids duplicate computation in self.experts.
        if self.is_sequence_parallel and not already_sequence_parallel:
            hidden_states = sequence_parallel_chunk(hidden_states)

        # MoERunner holds the gate (passed to FusedMoEFactory) and computes
        # the router logits itself, so nothing is precomputed here (matches
        # DeepseekV2MoE; `router_logits` is a placeholder).
        final_hidden_states = self.experts(
            hidden_states=hidden_states, router_logits=hidden_states
        )

        if self.is_sequence_parallel and not already_sequence_parallel:
            final_hidden_states = tensor_model_parallel_all_gather(
                final_hidden_states, 0
            )
            final_hidden_states = final_hidden_states[:num_tokens]

        return final_hidden_states.view(num_tokens, hidden_dim)

_init_mega_moe_experts(vllm_config, config, quant_config, prefix)

DeepGEMM MegaMoE, BF16xBF16 for an unquantized checkpoint and FP8xFP8 for a block-FP8 one: the kernel dispatches, runs the routed and shared experts and combines across the EP group.

Source code in vllm/models/glm5next/common/model.py
def _init_mega_moe_experts(
    self,
    vllm_config: VllmConfig,
    config: Glm5NextTextConfig,
    quant_config: QuantizationConfig | None,
    prefix: str,
) -> None:
    """DeepGEMM MegaMoE, BF16xBF16 for an unquantized checkpoint and
    FP8xFP8 for a block-FP8 one: the kernel dispatches, runs the routed and
    shared experts and combines across the EP group."""
    from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
        MegaMoEActivation,
    )
    from vllm.models.glm5next.nvidia.mega_moe import Glm5NextMegaMoEExperts

    if not vllm_config.parallel_config.enable_expert_parallel:
        raise NotImplementedError(
            "GLM-5.3 MegaMoE requires expert parallel. Enable it with "
            "--enable-expert-parallel, or pick a different moe backend."
        )
    if self.tp_size != 1:
        raise NotImplementedError(
            "GLM-5.3 MegaMoE requires tensor_parallel_size=1 (DP + EP)."
        )
    if (config.n_group or 1) > 1 or (config.topk_group or 1) > 1:
        raise NotImplementedError("GLM-5.3 MegaMoE requires one expert group.")
    weight_block_size = getattr(quant_config, "weight_block_size", None)
    if quant_config is not None and weight_block_size is None:
        raise NotImplementedError(
            "GLM-5.3 MegaMoE requires a BF16 or a block-FP8 checkpoint "
            "(quantization_config.weight_block_size)."
        )
    if self.n_physical_experts % self.ep_size != 0:
        raise ValueError(
            f"n_physical_experts={self.n_physical_experts} must be divisible "
            f"by ep_size={self.ep_size}."
        )
    self.mega_renormalize = config.norm_topk_prob
    self.mega_scoring_func = config.scoring_func
    self.experts = Glm5NextMegaMoEExperts(
        vllm_config,
        num_experts=self.n_physical_experts,
        num_local_experts=self.n_local_physical_experts,
        experts_start_idx=self.physical_expert_start,
        num_logical_experts=self.n_logical_experts,
        top_k=config.num_experts_per_tok,
        hidden_size=config.hidden_size,
        intermediate_size=config.moe_intermediate_size,
        weight_block_size=None
        if weight_block_size is None
        else tuple(weight_block_size),
        num_shared_experts=config.n_shared_experts or 0,
        activation=MegaMoEActivation("swiglu", clamp=config.swiglu_limit),
        prefix=f"{prefix}.experts",
    )

Glm5NextModel

Bases: Module, EagleModelMixin

Source code in vllm/models/glm5next/common/model.py
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
class Glm5NextModel(nn.Module, EagleModelMixin):
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__()

        config = vllm_config.model_config.hf_text_config
        _validate_supported_config(config)
        self.config = config
        self.use_mega_moe = (
            vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
        )

        self.vocab_size = config.vocab_size
        self.device = current_platform.device_type

        self.is_v32 = config.index_topk is not None
        if self.is_v32:
            topk_indices_buffer = torch.empty(
                vllm_config.scheduler_config.max_num_batched_tokens,
                get_sparse_topk_buffer_width(config),
                dtype=torch.int32,
                device=self.device,
            )
        else:
            # Full-MLA config (no kpool sparse indexer): no topk buffer.
            topk_indices_buffer = None

        if get_pp_group().is_first_rank:
            self.embed_tokens = VocabParallelEmbedding(
                config.vocab_size,
                config.hidden_size,
                prefix=f"{prefix}.embed_tokens",
            )
        else:
            self.embed_tokens = PPMissingLayer()

        def get_layer(prefix: str):
            layer_idx = int(prefix.rsplit(".", 1)[1])
            return Glm5NextDecoderLayer(
                vllm_config=vllm_config,
                config=config,
                layer_idx=layer_idx,
                prefix=prefix,
                topk_indices_buffer=topk_indices_buffer,
            )

        self.start_layer, self.end_layer, self.layers = make_layers(
            config.num_hidden_layers,
            get_layer,
            prefix=f"{prefix}.layers",
        )
        self._aux_post_op = MHCPostOp()
        # The active slice is fixed after construction; cache it so forward
        # doesn't rebuild the slice (a fresh list) every step.
        self._active_layers = self.layers[self.start_layer : self.end_layer]
        self.fused_shared_expert_layers = get_fused_shared_expert_layers(
            self,
            Glm5NextMoE,
            enable_eplb=vllm_config.parallel_config.enable_eplb,
        )

        if get_pp_group().is_last_rank:
            self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        else:
            self.norm = PPMissingLayer()

        self.is_sequence_parallel = (
            vllm_config.parallel_config.use_sequence_parallel_moe
        )

        world_size = get_tensor_model_parallel_world_size()
        assert config.num_attention_heads % world_size == 0, (
            "num_attention_heads must be divisible by world_size"
        )

    def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
        return self.embed_tokens(input_ids)

    def _aux_hidden_state(
        self,
        hidden_states: torch.Tensor,
        residual: torch.Tensor | None,
        post: torch.Tensor | None,
        comb: torch.Tensor | None,
    ) -> torch.Tensor:
        """Completed residual stream entering a layer, as one hidden vector.

        mHC layers defer their final `hc_post` into the next layer's fused
        pre-op, so `hidden_states` holds the raw layer output while the
        widened stream is completed here and contracted back to
        `hidden_size`. Non-mHC layers already return the summed stream.
        """
        if post is None:
            return hidden_states
        assert residual is not None and comb is not None
        completed = self._aux_post_op(hidden_states, residual, post, comb)
        return hc_contract(completed, self.config.hc_mult)

    def forward(
        self,
        input_ids: torch.Tensor | None,
        positions: torch.Tensor,
        intermediate_tensors: IntermediateTensors | None,
        inputs_embeds: torch.Tensor | None = None,
        **kwargs,
    ) -> torch.Tensor:
        if get_pp_group().is_first_rank:
            if inputs_embeds is not None:
                hidden_states = inputs_embeds
            else:
                hidden_states = self.embed_input_ids(input_ids)
            residual = None
            post = None
            comb = None
        else:
            assert intermediate_tensors is not None
            hidden_states = intermediate_tensors["hidden_states"]
            residual = intermediate_tensors["residual"]
            # post/comb (deferred mHC hc_post state) are not propagated across
            # PP ranks; the receiving rank's first mHC layer uses standalone pre.
            post = None
            comb = None

        full_num_tokens = positions.shape[0]
        if self.is_sequence_parallel:
            hidden_states = sp_shard(hidden_states)

        aux_hidden_states: list[torch.Tensor] = []
        for idx, layer in enumerate(self._active_layers, start=self.start_layer):
            if idx in self.aux_hidden_state_layers:
                aux_hidden_state = self._aux_hidden_state(
                    hidden_states, residual, post, comb
                )
                if self.is_sequence_parallel:
                    aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
                aux_hidden_states.append(aux_hidden_state)
            hidden_states, residual, post, comb = layer(
                positions, hidden_states, residual, post, comb
            )

        if not get_pp_group().is_last_rank:
            # PP is gated off for GLM-5.3-Flash (no make_empty_intermediate_tensors),
            # so this branch is not exercised. post/comb are the deferred
            # hc_post state of this rank's last mHC layer; a future PP path
            # would need to propagate them, but for now they are dropped (the
            # receiving rank's first layer would fall back to standalone pre).
            return IntermediateTensors(
                {"hidden_states": hidden_states, "residual": residual}
            )

        if self.end_layer in self.aux_hidden_state_layers:
            final_aux = self._aux_hidden_state(hidden_states, residual, post, comb)
            if self.is_sequence_parallel:
                final_aux = sp_all_gather(final_aux)[:full_num_tokens]
            aux_hidden_states.append(final_aux)

        if self.is_sequence_parallel:
            hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]

        hidden_states = self.norm(hidden_states)
        if aux_hidden_states:
            return hidden_states, aux_hidden_states
        return hidden_states

    def split_mhc_fns(self) -> None:
        for layer in self._active_layers:
            if isinstance(layer, Glm5NextDecoderLayer):
                layer.split_hc_fns()

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            (".gate_up_proj", ".gate_proj", 0),
            (".gate_up_proj", ".up_proj", 1),
            # MLA: fuse q_a_proj and kv_a_proj_with_mqa
            (".fused_qkv_a_proj", ".q_a_proj", 0),
            (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
            # KDA: merge q, k, v, b, f_a, g_a projections into one GEMM
            (".in_proj_qkvbfg_a", ".q_proj", 0),
            (".in_proj_qkvbfg_a", ".k_proj", 1),
            (".in_proj_qkvbfg_a", ".v_proj", 2),
            (".in_proj_qkvbfg_a", ".b_proj", 3),
            (".in_proj_qkvbfg_a", ".f_a_proj", 4),
            (".in_proj_qkvbfg_a", ".g_a_proj", 5),
            # KDA: fuse the q, k, v short convs into one conv1d
            (".conv1d", ".q_conv1d", 0),
            (".conv1d", ".k_conv1d", 1),
            (".conv1d", ".v_conv1d", 2),
        ]
        if _is_moe(self.config):
            # Params for weights, fp8 weight scales, fp8 activation scales
            # (param_name, weight_name, expert_id, shard_id)
            # EPLB: the mapping enumerates physical experts, so it must cover
            # the redundant replicas or their slots are never loaded.
            num_redundant_experts = next(
                (
                    layer.mlp.n_redundant_experts
                    for layer in self.layers
                    if isinstance(layer, Glm5NextDecoderLayer)
                    and isinstance(layer.mlp, Glm5NextMoE)
                ),
                0,
            )
            num_fused_shared = _num_fused_shared_experts(
                self.config.n_shared_experts,
                bool(self.fused_shared_expert_layers),
            )
            expert_params_mapping = fused_moe_make_expert_params_mapping(
                self,
                ckpt_gate_proj_name="gate_proj",
                ckpt_down_proj_name="down_proj",
                ckpt_up_proj_name="up_proj",
                # NOTE: global despite per-layer FSE; non-fused layers skip the extras.
                num_experts=self.config.n_routed_experts + num_fused_shared,
                num_redundant_experts=num_redundant_experts,
                routed_experts_prefix="" if self.use_mega_moe else "routed_experts",
            )
        else:
            expert_params_mapping = []
        params_dict = dict(self.named_parameters())
        loaded_params: set[str] = set()

        # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
        # ``kv_a_proj_with_mqa``; pad them with zeros for the model shape.
        kv_a_pad_size = 0
        if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
            kv_a_pad_size = self.config.qk_rope_head_dim

        for args in weights:
            name, loaded_weight = args[:2]
            kwargs: dict = args[2] if len(args) > 2 else {}
            if "rotary_emb.inv_freq" in name:
                continue

            spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
            if spec_layer is not None:
                continue  # skip spec decode layers for main model
            if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
                # Models trained using ColossalAI may include these tensors in
                # the checkpoint. Skip them.
                continue
            if self.use_mega_moe and ".shared_experts." in name:
                mlp = self.get_submodule(name.partition(".shared_experts.")[0])
                loaded_params.add(
                    mlp.experts.load_shared_expert_weight(name, loaded_weight)
                )
                continue
            # Only parse the layer index when FSE is enabled.
            if (
                "mlp.shared_experts." in name
                and bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
                and extract_layer_index(name) in self.fused_shared_expert_layers
            ):
                name = _fused_shared_expert_name(name, self.config.n_routed_experts)

            name, loaded_weight = _mla_proj_checkpoint_tensor(
                name, loaded_weight, params_dict, self.config, kv_a_pad_size
            )

            for param_name, weight_name, shard_id in stacked_params_mapping:
                if weight_name not in name:
                    continue
                # We have mlp.experts[0].gate_proj in the checkpoint.
                # Since we handle the experts below in expert_params_mapping,
                # we need to skip here BEFORE we update the name, otherwise
                # name will be updated to mlp.experts[0].gate_up_proj, which
                # will then be updated below in expert_params_mapping
                # for mlp.experts[0].gate_gate_up_proj, which breaks load.
                if ("mlp.experts." in name) and name not in params_dict:
                    continue
                name_mapped = name.replace(weight_name, param_name)
                # QKV fusion: skip if fused module doesn't exist in model
                if param_name == ".fused_qkv_a_proj" and name_mapped not in params_dict:
                    continue
                name = name_mapped
                # Skip loading extra bias for GPTQ models.
                if name.endswith(".bias") and name not in params_dict:
                    continue
                if is_pp_missing_parameter(name, self):
                    continue
                param = params_dict[name]
                weight_loader = param.weight_loader
                weight_loader(param, loaded_weight, shard_id)
                break
            else:
                is_expert_weight = False
                for (
                    param_name,
                    weight_name,
                    expert_id,
                    expert_shard_id,
                ) in expert_params_mapping:
                    if weight_name not in name:
                        continue
                    # A checkpoint expert may map to several physical replicas
                    # under EPLB; keep `name` intact and try the next entry
                    # when this physical expert is not local to the rank.
                    is_expert_weight = True
                    name_mapped = name.replace(weight_name, param_name)
                    if is_pp_missing_parameter(name_mapped, self):
                        continue
                    param = params_dict[name_mapped]
                    weight_loader = param.weight_loader
                    success = weight_loader(
                        param,
                        loaded_weight,
                        name_mapped,
                        expert_id=expert_id,
                        shard_id=expert_shard_id,
                        return_success=True,
                    )
                    if success:
                        name = name_mapped
                        break
                else:
                    if is_expert_weight:
                        continue
                    # Skip loading extra bias for GPTQ models.
                    if (
                        name.endswith(".bias")
                        and name not in params_dict
                        and not _is_linear_attn(self.config)
                    ):  # noqa: E501
                        continue
                    # Remapping the name of FP8 kv-scale.
                    remapped_name = maybe_remap_kv_scale_name(name, params_dict)
                    if remapped_name is None:
                        continue
                    name = remapped_name
                    if is_pp_missing_parameter(name, self):
                        continue

                    param = params_dict[name]
                    weight_loader = getattr(
                        param, "weight_loader", default_weight_loader
                    )
                    weight_loader(param, loaded_weight, **kwargs)
            loaded_params.add(name)
        return loaded_params

_aux_hidden_state(hidden_states, residual, post, comb)

Completed residual stream entering a layer, as one hidden vector.

mHC layers defer their final hc_post into the next layer's fused pre-op, so hidden_states holds the raw layer output while the widened stream is completed here and contracted back to hidden_size. Non-mHC layers already return the summed stream.

Source code in vllm/models/glm5next/common/model.py
def _aux_hidden_state(
    self,
    hidden_states: torch.Tensor,
    residual: torch.Tensor | None,
    post: torch.Tensor | None,
    comb: torch.Tensor | None,
) -> torch.Tensor:
    """Completed residual stream entering a layer, as one hidden vector.

    mHC layers defer their final `hc_post` into the next layer's fused
    pre-op, so `hidden_states` holds the raw layer output while the
    widened stream is completed here and contracted back to
    `hidden_size`. Non-mHC layers already return the summed stream.
    """
    if post is None:
        return hidden_states
    assert residual is not None and comb is not None
    completed = self._aux_post_op(hidden_states, residual, post, comb)
    return hc_contract(completed, self.config.hc_mult)

_fused_shared_expert_name(name, n_routed_experts)

Point a checkpoint mlp.shared_experts.* tensor at the fused MoE's shared-expert slot, which follows the routed experts; other names are returned unchanged.

Source code in vllm/models/glm5next/common/model.py
def _fused_shared_expert_name(name: str, n_routed_experts: int) -> str:
    """Point a checkpoint `mlp.shared_experts.*` tensor at the fused MoE's
    shared-expert slot, which follows the routed experts; other names are
    returned unchanged."""
    return name.replace("mlp.shared_experts.", f"mlp.experts.{n_routed_experts}.", 1)

_fused_shared_experts_tuned(parallel_config)

AITER has fused-MoE configs tuned for the fused shared-expert shape (one more expert and one more top-k slot than the routed MoE) only on gfx950, with every expert on each rank and its weights split by TP4 or TP8. Data, prefill context and expert parallelism change that split, so any other GPU or parallel layout would run untuned fallback kernels.

Source code in vllm/models/glm5next/common/model.py
def _fused_shared_experts_tuned(parallel_config: ParallelConfig) -> bool:
    """AITER has fused-MoE configs tuned for the fused shared-expert shape
    (one more expert and one more top-k slot than the routed MoE) only on
    gfx950, with every expert on each rank and its weights split by TP4 or
    TP8. Data, prefill context and expert parallelism change that split, so
    any other GPU or parallel layout would run untuned fallback kernels."""
    from vllm.platforms.rocm import on_gfx950

    reasons: list[str] = []
    if not on_gfx950():
        reasons.append("the GPU is not gfx950")
    if parallel_config.tensor_parallel_size not in (4, 8):
        reasons.append(
            f"tensor_parallel_size is {parallel_config.tensor_parallel_size}"
        )
    if parallel_config.data_parallel_size != 1:
        reasons.append(f"data_parallel_size is {parallel_config.data_parallel_size}")
    if parallel_config.prefill_context_parallel_size != 1:
        reasons.append(
            "prefill_context_parallel_size is "
            f"{parallel_config.prefill_context_parallel_size}"
        )
    if parallel_config.enable_expert_parallel:
        reasons.append("expert parallelism is enabled")

    if not reasons:
        return True
    logger.warning_once(
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is ignored for GLM-5.3-Flash: "
        "%s. AITER has tuned configs for its fused shared-expert MoE only on "
        "gfx950 at TP4 and TP8, without data, prefill context or expert "
        "parallelism. Running the shared experts as a separate MLP.",
        "; ".join(reasons),
    )
    return False

_mla_proj_checkpoint_tensor(name, tensor, params_dict, config, kv_a_pad_size)

A checkpoint tensor of an MLA projection, in the form its parameter loads: Quark's weight_scale under the weight_scale_inv name, and a NoPE kv_a_proj_with_mqa with zero rows (and scale blocks) for the RoPE part the checkpoint omits. Each tensor is adapted on its own, so a weight and its scale can arrive in any order and in different calls.

Source code in vllm/models/glm5next/common/model.py
def _mla_proj_checkpoint_tensor(
    name: str,
    tensor: torch.Tensor,
    params_dict: dict[str, nn.Parameter],
    config: Glm5NextTextConfig,
    kv_a_pad_size: int,
) -> tuple[str, torch.Tensor]:
    """A checkpoint tensor of an MLA projection, in the form its parameter
    loads: Quark's `weight_scale` under the `weight_scale_inv` name, and a
    NoPE `kv_a_proj_with_mqa` with zero rows (and scale blocks) for the RoPE
    part the checkpoint omits. Each tensor is adapted on its own, so a weight
    and its scale can arrive in any order and in different calls."""
    suffix = next((s for s in _FP8_ATTN_PROJS if s in name), None)
    if suffix is None:
        return name, tensor
    prefix = name.rsplit(suffix, 1)[0]
    scale_name = f"{prefix}.{_FP8_ATTN_PROJS[suffix]}.weight_scale_inv"
    if scale_name not in params_dict:  # no q_lora_rank: kv_a is not fused
        scale_name = f"{prefix}{suffix}weight_scale_inv"
    is_fp8_proj = scale_name in params_dict  # see `bf16_linear`
    if is_fp8_proj and name.endswith(".weight_scale"):
        name = f"{name}_inv"
    if kv_a_pad_size > 0 and suffix == ".kv_a_proj_with_mqa.":
        if name.endswith(".weight"):
            pad = tensor.new_zeros(kv_a_pad_size, *tensor.shape[1:])
            tensor = torch.cat([tensor, pad])
        elif name.endswith((".weight_scale_inv", ".weight_scale")) and tensor.ndim == 2:
            # block scales; Quark names them `weight_scale` (not renamed above
            # when its own layer, with a `weight_scale` parameter, loads them)
            block_n = cdiv(config.kv_lora_rank, tensor.shape[0])
            rows = cdiv(config.kv_lora_rank + kv_a_pad_size, block_n)
            pad = tensor.new_ones(rows - tensor.shape[0], *tensor.shape[1:])
            tensor = torch.cat([tensor, pad])
    return name, tensor

_num_fused_shared_experts(n_shared_experts, enabled)

Expert slots the fused MoE appends for the shared expert; must match the num_fused_shared_experts that FusedMoE allocates.

Source code in vllm/models/glm5next/common/model.py
def _num_fused_shared_experts(n_shared_experts: int | None, enabled: bool) -> int:
    """Expert slots the fused MoE appends for the shared expert; must match the
    `num_fused_shared_experts` that `FusedMoE` allocates."""
    if not enabled or n_shared_experts is None:
        return 0
    if n_shared_experts > 1:
        raise NotImplementedError(
            "Fused shared-expert loading supports only 1 shared expert per "
            f"layer, but config.n_shared_experts is {n_shared_experts}. Set "
            "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 to run the shared "
            "experts as a separate MLP."
        )
    return n_shared_experts

_validate_supported_config(config)

Reject checkpoints using config options this implementation lacks.

The kpool indexer kernels always keep the incomplete trailing pool, so a checkpoint asking otherwise would be served silently wrong.

Source code in vllm/models/glm5next/common/model.py
def _validate_supported_config(config: Glm5NextTextConfig) -> None:
    """Reject checkpoints using config options this implementation lacks.

    The kpool indexer kernels always keep the incomplete trailing pool, so a
    checkpoint asking otherwise would be served silently wrong.
    """
    if config.index_topk is not None and not config.index_kpool_always_select_tail:
        raise NotImplementedError(
            "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
        )

get_sparse_topk_buffer_width(config)

Reserve the incomplete pool tail and align sparse MLA tiles.

Source code in vllm/models/glm5next/common/model.py
def get_sparse_topk_buffer_width(config: Glm5NextTextConfig) -> int:
    """Reserve the incomplete pool tail and align sparse MLA tiles."""
    topk = config.index_topk
    assert topk is not None
    kpool = config.index_kpool
    assert kpool is not None
    return round_up(topk + kpool - 1, 128)