Skip to content

vllm.models.minimax_m3.nvidia.specialized_decode.kernels.moe

The moe stage of the MiniMax-M3 specialized decode: the fused NVFP4 routed + MXFP8 shared-expert engine (CuTe DSL, kernel.py), its host (host.py), the weight views and mode-Q prep (weights.py) and the stage wrapper (stage.py). Importing this package does not import CuTe DSL; the kernel module loads on first build.

Modules:

  • host –

    Host side of the fused MoE engine (kernel.py, 1..8 tokens): router, NVFP4 routed

  • kernel –

    The fused MiniMax-M3 MoE engine for 1..8 decode tokens on one TP rank, one CuTe DSL kernel (m3_moe_kernel).

  • stage –

    The moe stage: the fused MoE engine (TP4, 1..8 rows, mode Q) on the post's

  • weights –

    MoE engine operands from vLLM's loaded MiniMax-M3 MoE layer, and the mode-Q FC2

Classes:

Functions:

  • default_n_shared –

    Shared-expert CTAs for rows tokens on the live sm_count

  • engine_config –

    The engine's trace-time config for the TP4 mode-Q build.

  • from_vllm_moe –

    MoeWeights from a loaded MiniMaxM3MoE (decoder_layer.block_sparse_moe) on

  • prep_mode_q –

    Mode Q FC2 prep as new tensors (idempotent): fc31_scale_c = g1_alphas x 16,

FusedMoeStage

See the module docstring.

Methods:

  • __call__ –

    One engine launch. hidden: bf16 [rows, 6144] (default: the hand-off's

  • __init__ –

    handoff: the post's allreduce_norm.MoeHandoff (routes, x4, normed

  • warmup –

    Compile with one eager call (no push). Must run before CUDA-graph capture,

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
class FusedMoeStage:
    """See the module docstring."""

    stage = "moe"

    def __init__(
        self,
        cfg: FusedMoeConfig,
        weights: MoeWeights,
        *,
        transport=None,
        handoff=None,
        rt_tab: torch.Tensor | None = None,
        x4: torch.Tensor | None = None,
        x4sf: torch.Tensor | None = None,
        co_resident: Sequence[_arch.GridUse] = (),
        device: torch.device | None = None,
    ):
        """`handoff`: the post's `allreduce_norm.MoeHandoff` (routes, x4, normed
        rows). Tests without a post pass
        `rt_tab` (ext_route=2) and `x4` / `x4sf` (uint8 `[16][3072]` / `[17][384]`,
        row 16 = fp32 s_t) instead."""
        self.cfg = cfg
        dev = torch.device(device) if device is not None else torch.device("cuda")
        if (
            dev.index is None
        ):  # "cuda" -> "cuda:<current>": the operand checks compare devices exactly
            dev = torch.device("cuda", torch.accelerator.current_device_index())
        self.device = dev
        self.spec = _arch.device_spec(dev.index)
        if cfg.fc1_per_token:
            weights = prep_mode_q(weights)
            if weights.fc2_enc != FC2_SCALE_Q:
                raise ValueError("moe: mode Q needs the FC2 scale-16 weight prep")
        weights.validate()
        self.weights = weights
        nsh = (
            cfg.n_shared
            if cfg.n_shared is not None
            else default_n_shared(cfg.rows, self.spec.sm_count)
        )
        self.n_shared = nsh
        self.num_ctas = self.spec.sm_count - nsh
        if self.num_ctas < H.MIN_CTAS:
            raise _arch.ResidencyError(
                f"moe: {self.spec.sm_count} SMs leave {self.num_ctas} routed CTAs "
                f"(< {H.MIN_CTAS}) beside {nsh} shared CTAs"
            )
        self.push_ws = None
        if cfg.push:
            if transport is None:
                raise ValueError("moe: push=True needs a PeerTransport")
            self.push_ws = push_workspace(transport, bucket(cfg.rows), "g3")
        self.state = H.M3MoeState(
            dev,
            cfg.rows,
            self.num_ctas,
            shared=True,
            n_shared=nsh,
            extra_cfg=engine_config(cfg),
            push_ws=self.push_ws,
        )
        self._check_residency(co_resident)
        self.handoff = handoff
        route_lines = rctr = None
        if cfg.ext_route == 1:
            if handoff is None:
                raise ValueError("moe: ext_route=1 needs the post's MoeHandoff")
            if handoff.rows != cfg.rows:
                raise ValueError(
                    f"moe: handoff rows {handoff.rows} != config rows {cfg.rows}"
                )
            route_lines, rctr = handoff.route, handoff.rctr
        elif rt_tab is None:
            raise ValueError("moe: ext_route=2 needs rt_tab")
        if handoff is not None:
            x4, x4sf = (
                handoff.x4_q,
                (handoff.x4_sf_full if cfg.fc1_per_token else handoff.x4_sf),
            )
        if x4 is None or x4sf is None:
            raise ValueError(
                "moe: the engine reads the post's NVFP4 FC1 input: pass handoff or x4 "
                "/ x4sf"
            )
        self._rt_tab = rt_tab
        w = weights
        self.layer = self.state.layer(  # type: ignore[misc]
            *w.expert_args(),
            router_weight=w.router,
            fc31_input_scale=w.a1_gscale,
            shared_weights=w.shared(),
            route_lines=route_lines,
            rt_tab=rt_tab,
            rctr=rctr,
            x4=x4,
            x4sf=x4sf,
        )
        self.arp = self._peers = None
        if self.push_ws is not None:
            self.arp, self._peers = H.push_arp(self.push_ws, dev)
        self._warm = False

    # ------------------------------------------------------------------ residency (fail
    # closed)
    def grid_use(self) -> _arch.GridUse:
        st = self.state
        # maxnreg (the b1_x4 builds' register cap) bounds the engine's registers; 0 =
        # uncapped, not counted
        return _arch.GridUse(
            "moe.engine",
            st.grid_ctas,
            st.smem_bytes_upper_bound,
            st.threads,
            tmem_columns=_arch.TMEM_COLUMNS,
            regs_per_thread=st.maxnreg,
        )

    def _check_residency(self, co_resident):
        st, spec = self.state, self.spec
        if st.grid_ctas != spec.sm_count:
            # one persistent CTA per SM: more would deadlock, fewer leaves SMs to a
            # co-running kernel's CTAs
            raise _arch.ResidencyError(
                f"moe: grid {st.grid_ctas} != {spec.sm_count} SMs (one CTA per SM)"
            )
        # The CuTe DSL build exposes no function handle for
        # cudaOccupancyMaxActiveBlocksPerMultiprocessor, so the occupancy is the
        # resource bound (smem, threads, all 512 TMEM columns, the maxnreg register
        # cap).
        occ = _arch.ctas_per_sm(
            spec,
            smem_bytes=st.smem_bytes_upper_bound,
            threads=st.threads,
            tmem_columns=_arch.TMEM_COLUMNS,
            regs_per_thread=st.maxnreg,
        )
        # every engine CTA polls the others (FC1 -> FC2 records, B images, shared-expert
        # h / y_sh)
        _arch.check_persistent_grid("moe.engine", st.grid_ctas, occ, spec)
        if self.cfg.ext_route == 1 and not co_resident:
            raise _arch.ResidencyError(
                "moe: ext_route=1 polls the post's route lines before its grid wait; "
                "pass "
                "the post's GridUse as co_resident so co-residency can be checked"
            )
        if co_resident:
            _arch.check_co_resident([self.grid_use(), *co_resident], spec)

    # ------------------------------------------------------------------ compile / run
    def _seed_publish(self):
        """The compile call consumes one route publish: align the post's row counters
        with the engine's epochs and
        publish experts 0..3 at weight 0.25 with tag = epoch + 1 (what the post's next
        publish would carry)."""
        st, hd = self.state, self.handoff
        ep = st.epochs[: st.num_ctas].cpu()
        if ep.numel() == 0 or not bool((ep == ep[0]).all()):
            raise RuntimeError(
                "moe: engine epochs not uniform before the compile call "
                f"{ep.unique().tolist()}"
            )
        tag = int(ep[0]) + 1
        t = self.cfg.rows
        ids = torch.arange(4, dtype=torch.int64, device=self.device)
        words = (tag << 32) | (ids << 16) | SEED_WEIGHT_BF16
        hd.route.zero_()
        hd.route[:, :t, :4] = words.view(1, 1, 4)
        hd.rctr.view(-1, RCTR_STRIDE)[:, 0] = tag
        return tag

    def warmup(self):
        """Compile with one eager call (no push). Must run before CUDA-graph capture,
        outside any PDL chain."""
        if torch.cuda.is_current_stream_capturing():
            raise RuntimeError("moe: warmup during graph capture")
        tag = None
        if self.cfg.ext_route == 1:
            tag = self._seed_publish()
        hidden = (
            self.handoff.normed
            if self.handoff is not None
            else torch.zeros(
                self.cfg.rows, H.HIDDEN, dtype=torch.bfloat16, device=self.device
            )
        )
        out = torch.empty(
            self.cfg.rows, H.HIDDEN, dtype=torch.bfloat16, device=self.device
        )
        self.layer(hidden, out=out)
        torch.accelerator.synchronize(self.device)
        if tag is not None:
            st = self.state
            ep = st.epochs[: st.num_ctas].cpu()
            if not bool((ep == tag).all()):
                raise RuntimeError(
                    "moe: after the compile call the engine epochs "
                    f"{ep.unique().tolist()} != route "
                    f"tag {tag}; the next post publish would never be consumed"
                )
            rd = st.route_dbg[: self.cfg.rows].cpu()
            if not bool(rd[:, 4:8].any(dim=1).all()):
                raise RuntimeError(
                    "moe: the compile call fell back to the give-up route (no publish "
                    "seen)"
                )
        self._warm = True

    def __call__(
        self,
        hidden: torch.Tensor | None = None,
        out: torch.Tensor | None = None,
        push: bool = False,
        rt_tab: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """One engine launch. `hidden`: bf16 [rows, 6144] (default: the hand-off's
        normed rows, the shared-expert
        input). `push`: also push the output into the boundary exchange (only when the
        next consumer polls it,
        e.g. the next layer's kernel front; an unconsumed push would collide with the
        next step's)."""
        if not self._warm:
            raise RuntimeError("moe: call warmup() first (it compiles)")
        if hidden is None:
            hidden = self.handoff.normed
        if push and self.arp is None:
            raise ValueError(
                "moe: push requested but the stage was built without a push workspace"
            )
        return self.layer(
            hidden, out=out, arp=self.arp if push else None, rt_tab=rt_tab
        )

__call__(hidden=None, out=None, push=False, rt_tab=None)

One engine launch. hidden: bf16 [rows, 6144] (default: the hand-off's normed rows, the shared-expert input). push: also push the output into the boundary exchange (only when the next consumer polls it, e.g. the next layer's kernel front; an unconsumed push would collide with the next step's).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def __call__(
    self,
    hidden: torch.Tensor | None = None,
    out: torch.Tensor | None = None,
    push: bool = False,
    rt_tab: torch.Tensor | None = None,
) -> torch.Tensor:
    """One engine launch. `hidden`: bf16 [rows, 6144] (default: the hand-off's
    normed rows, the shared-expert
    input). `push`: also push the output into the boundary exchange (only when the
    next consumer polls it,
    e.g. the next layer's kernel front; an unconsumed push would collide with the
    next step's)."""
    if not self._warm:
        raise RuntimeError("moe: call warmup() first (it compiles)")
    if hidden is None:
        hidden = self.handoff.normed
    if push and self.arp is None:
        raise ValueError(
            "moe: push requested but the stage was built without a push workspace"
        )
    return self.layer(
        hidden, out=out, arp=self.arp if push else None, rt_tab=rt_tab
    )

__init__(cfg, weights, *, transport=None, handoff=None, rt_tab=None, x4=None, x4sf=None, co_resident=(), device=None)

handoff: the post's allreduce_norm.MoeHandoff (routes, x4, normed rows). Tests without a post pass rt_tab (ext_route=2) and x4 / x4sf (uint8 [16][3072] / [17][384], row 16 = fp32 s_t) instead.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def __init__(
    self,
    cfg: FusedMoeConfig,
    weights: MoeWeights,
    *,
    transport=None,
    handoff=None,
    rt_tab: torch.Tensor | None = None,
    x4: torch.Tensor | None = None,
    x4sf: torch.Tensor | None = None,
    co_resident: Sequence[_arch.GridUse] = (),
    device: torch.device | None = None,
):
    """`handoff`: the post's `allreduce_norm.MoeHandoff` (routes, x4, normed
    rows). Tests without a post pass
    `rt_tab` (ext_route=2) and `x4` / `x4sf` (uint8 `[16][3072]` / `[17][384]`,
    row 16 = fp32 s_t) instead."""
    self.cfg = cfg
    dev = torch.device(device) if device is not None else torch.device("cuda")
    if (
        dev.index is None
    ):  # "cuda" -> "cuda:<current>": the operand checks compare devices exactly
        dev = torch.device("cuda", torch.accelerator.current_device_index())
    self.device = dev
    self.spec = _arch.device_spec(dev.index)
    if cfg.fc1_per_token:
        weights = prep_mode_q(weights)
        if weights.fc2_enc != FC2_SCALE_Q:
            raise ValueError("moe: mode Q needs the FC2 scale-16 weight prep")
    weights.validate()
    self.weights = weights
    nsh = (
        cfg.n_shared
        if cfg.n_shared is not None
        else default_n_shared(cfg.rows, self.spec.sm_count)
    )
    self.n_shared = nsh
    self.num_ctas = self.spec.sm_count - nsh
    if self.num_ctas < H.MIN_CTAS:
        raise _arch.ResidencyError(
            f"moe: {self.spec.sm_count} SMs leave {self.num_ctas} routed CTAs "
            f"(< {H.MIN_CTAS}) beside {nsh} shared CTAs"
        )
    self.push_ws = None
    if cfg.push:
        if transport is None:
            raise ValueError("moe: push=True needs a PeerTransport")
        self.push_ws = push_workspace(transport, bucket(cfg.rows), "g3")
    self.state = H.M3MoeState(
        dev,
        cfg.rows,
        self.num_ctas,
        shared=True,
        n_shared=nsh,
        extra_cfg=engine_config(cfg),
        push_ws=self.push_ws,
    )
    self._check_residency(co_resident)
    self.handoff = handoff
    route_lines = rctr = None
    if cfg.ext_route == 1:
        if handoff is None:
            raise ValueError("moe: ext_route=1 needs the post's MoeHandoff")
        if handoff.rows != cfg.rows:
            raise ValueError(
                f"moe: handoff rows {handoff.rows} != config rows {cfg.rows}"
            )
        route_lines, rctr = handoff.route, handoff.rctr
    elif rt_tab is None:
        raise ValueError("moe: ext_route=2 needs rt_tab")
    if handoff is not None:
        x4, x4sf = (
            handoff.x4_q,
            (handoff.x4_sf_full if cfg.fc1_per_token else handoff.x4_sf),
        )
    if x4 is None or x4sf is None:
        raise ValueError(
            "moe: the engine reads the post's NVFP4 FC1 input: pass handoff or x4 "
            "/ x4sf"
        )
    self._rt_tab = rt_tab
    w = weights
    self.layer = self.state.layer(  # type: ignore[misc]
        *w.expert_args(),
        router_weight=w.router,
        fc31_input_scale=w.a1_gscale,
        shared_weights=w.shared(),
        route_lines=route_lines,
        rt_tab=rt_tab,
        rctr=rctr,
        x4=x4,
        x4sf=x4sf,
    )
    self.arp = self._peers = None
    if self.push_ws is not None:
        self.arp, self._peers = H.push_arp(self.push_ws, dev)
    self._warm = False

_seed_publish()

The compile call consumes one route publish: align the post's row counters with the engine's epochs and publish experts 0..3 at weight 0.25 with tag = epoch + 1 (what the post's next publish would carry).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def _seed_publish(self):
    """The compile call consumes one route publish: align the post's row counters
    with the engine's epochs and
    publish experts 0..3 at weight 0.25 with tag = epoch + 1 (what the post's next
    publish would carry)."""
    st, hd = self.state, self.handoff
    ep = st.epochs[: st.num_ctas].cpu()
    if ep.numel() == 0 or not bool((ep == ep[0]).all()):
        raise RuntimeError(
            "moe: engine epochs not uniform before the compile call "
            f"{ep.unique().tolist()}"
        )
    tag = int(ep[0]) + 1
    t = self.cfg.rows
    ids = torch.arange(4, dtype=torch.int64, device=self.device)
    words = (tag << 32) | (ids << 16) | SEED_WEIGHT_BF16
    hd.route.zero_()
    hd.route[:, :t, :4] = words.view(1, 1, 4)
    hd.rctr.view(-1, RCTR_STRIDE)[:, 0] = tag
    return tag

warmup()

Compile with one eager call (no push). Must run before CUDA-graph capture, outside any PDL chain.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def warmup(self):
    """Compile with one eager call (no push). Must run before CUDA-graph capture,
    outside any PDL chain."""
    if torch.cuda.is_current_stream_capturing():
        raise RuntimeError("moe: warmup during graph capture")
    tag = None
    if self.cfg.ext_route == 1:
        tag = self._seed_publish()
    hidden = (
        self.handoff.normed
        if self.handoff is not None
        else torch.zeros(
            self.cfg.rows, H.HIDDEN, dtype=torch.bfloat16, device=self.device
        )
    )
    out = torch.empty(
        self.cfg.rows, H.HIDDEN, dtype=torch.bfloat16, device=self.device
    )
    self.layer(hidden, out=out)
    torch.accelerator.synchronize(self.device)
    if tag is not None:
        st = self.state
        ep = st.epochs[: st.num_ctas].cpu()
        if not bool((ep == tag).all()):
            raise RuntimeError(
                "moe: after the compile call the engine epochs "
                f"{ep.unique().tolist()} != route "
                f"tag {tag}; the next post publish would never be consumed"
            )
        rd = st.route_dbg[: self.cfg.rows].cpu()
        if not bool(rd[:, 4:8].any(dim=1).all()):
            raise RuntimeError(
                "moe: the compile call fell back to the give-up route (no publish "
                "seen)"
            )
    self._warm = True

MoeWeights dataclass

Methods:

  • expert_args –

    The 11 positional operands of M3MoeState.layer.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/weights.py
@dataclass(frozen=True)
class MoeWeights:
    w13: torch.Tensor
    w13_scale: torch.Tensor
    w2: torch.Tensor
    w2_scale: torch.Tensor
    g1_alphas: torch.Tensor
    g1_scale_c: torch.Tensor
    g2_alphas: torch.Tensor
    act_alpha: torch.Tensor
    act_beta: torch.Tensor
    act_clamp: torch.Tensor
    a1_gscale: torch.Tensor
    router: torch.Tensor
    bias: torch.Tensor
    shared_gu_w: torch.Tensor
    shared_gu_s: torch.Tensor
    shared_dn_w: torch.Tensor
    shared_dn_s: torch.Tensor
    fc2_enc: float  # the FC2 input encode scale folded into g1_scale_c / g2_alphas

    def expert_args(self) -> tuple:
        """The 11 positional operands of `M3MoeState.layer`."""
        return (
            self.w13,
            self.w13_scale,
            self.w2,
            self.w2_scale,
            self.g1_alphas,
            self.g1_scale_c,
            self.act_alpha,
            self.act_beta,
            self.act_clamp,
            self.g2_alphas,
            self.bias,
        )

    def shared(self) -> tuple:
        return (self.shared_gu_w, self.shared_gu_s, self.shared_dn_w, self.shared_dn_s)

    def validate(self) -> None:
        e, i, h = NUM_EXPERTS, INTER, HIDDEN
        want = {
            "w13": (self.w13, (e, 2 * i, h // 2)),
            "w13_scale": (self.w13_scale, (e, 2 * i, h // 16)),
            "w2": (self.w2, (e, h, i // 2)),
            "w2_scale": (self.w2_scale, (e, h, i // 16)),
            "router": (self.router, (e, h)),
            "shared_gu_w": (self.shared_gu_w, (2 * i, h)),
            "shared_dn_w": (self.shared_dn_w, (h, i)),
        }
        for name, (t, shape) in want.items():
            if tuple(t.shape) != shape or not t.is_contiguous():
                raise MoeWeightError(
                    f"MoE operand {name}: {tuple(t.shape)} (contiguous "
                    f"{t.is_contiguous()}), "
                    f"expected contiguous {shape}"
                )
        for name in ("w13", "w13_scale", "w2", "w2_scale"):
            if getattr(self, name).element_size() != 1:
                raise MoeWeightError(
                    f"MoE operand {name} must be 1-byte (NVFP4 codes / e4m3 scales)"
                )
        for name in (
            "g1_alphas",
            "g1_scale_c",
            "g2_alphas",
            "act_alpha",
            "act_beta",
            "act_clamp",
            "bias",
        ):
            t = getattr(self, name)
            if t.dtype != torch.float32 or t.numel() != e:
                raise MoeWeightError(
                    f"MoE scalar {name}: {t.dtype} x {t.numel()}, expected float32 x "
                    f"{e}"
                )
        if self.a1_gscale.dtype != torch.float32 or self.a1_gscale.numel() != 1:
            raise MoeWeightError("MoE a1_gscale must be float32 [1]")
        if (
            self.shared_gu_w.dtype != torch.float8_e4m3fn
            or self.shared_dn_w.dtype != torch.float8_e4m3fn
        ):
            raise MoeWeightError("shared expert weights must be float8_e4m3fn (MXFP8)")
        if (
            self.shared_gu_s.numel() != 2 * i * h // 32
            or self.shared_dn_s.numel() != h * i // 32
        ):
            raise MoeWeightError(
                "shared expert scales must be the 128x4-swizzled UE8M0 [N, K / 32] "
                "buffers"
            )

expert_args()

The 11 positional operands of M3MoeState.layer.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/weights.py
def expert_args(self) -> tuple:
    """The 11 positional operands of `M3MoeState.layer`."""
    return (
        self.w13,
        self.w13_scale,
        self.w2,
        self.w2_scale,
        self.g1_alphas,
        self.g1_scale_c,
        self.act_alpha,
        self.act_beta,
        self.act_clamp,
        self.g2_alphas,
        self.bias,
    )

default_n_shared(rows, sm_count)

Shared-expert CTAs for rows tokens on the live sm_count (geometry.moe_n_shared: a bytes-per-CTA model, or a measured override).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def default_n_shared(rows: int, sm_count: int) -> int:
    """Shared-expert CTAs for `rows` tokens on the live `sm_count`
    (`geometry.moe_n_shared`: a bytes-per-CTA
    model, or a measured override)."""
    return geometry.moe_n_shared(rows, sm_count)

engine_config(cfg)

The engine's trace-time config for the TP4 mode-Q build.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/stage.py
def engine_config(cfg: FusedMoeConfig) -> dict:
    """The engine's trace-time config for the TP4 mode-Q build."""
    t = cfg.rows
    c = {"sf2_w6": 1, "b2buf": 1, "stages": 7} if t > 4 else {}
    if (
        t > 2
        # sf2_w6 moves the FC2 scale gather to warp 8, which exists only above 2 rows
        # (more builder warps)
    ):
        c["sf2_w6"] = 1
    c.update(
        early_trig=1, sfa_bulk=1, sh_nst=4, sh_pf=0, roll_fc2=1, b2img=1, sh_trig=1
    )
    c.update(ext_route=cfg.ext_route, xstage=0, pad_x=1, b1_x4=1)
    if cfg.ext_route == 1:
        c.update(route_repl=ROUTE_REPL, route_poll_ns=64, route_maxt=t)
    if cfg.fc1_per_token:
        c["pt5"] = 1
    c.update(dict(cfg.extra))
    return c

from_vllm_moe(moe)

MoeWeights from a loaded MiniMaxM3MoE (decoder_layer.block_sparse_moe) on the FLASHINFER_TRTLLM NVFP4 backend. Raises MoeWeightError with the reason when the layer is in any other form.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/weights.py
def from_vllm_moe(moe) -> MoeWeights:
    """MoeWeights from a loaded `MiniMaxM3MoE` (`decoder_layer.block_sparse_moe`) on
    the FLASHINFER_TRTLLM NVFP4
    backend. Raises `MoeWeightError` with the reason when the layer is in any other
    form."""
    ex = getattr(moe.experts, "routed_experts", moe.experts)
    qm = getattr(ex, "quant_method", None)
    kern = getattr(qm, "moe_kernel", None)
    fe = getattr(kern, "fused_experts", None)
    if fe is None or not hasattr(fe, "g1_scale_c"):
        raise MoeWeightError(
            "routed experts are not on the TRTLLM-Gen NVFP4 path (no "
            "fused_experts.g1_scale_c); "
            "serve with the FLASHINFER_TRTLLM NvFP4 MoE backend"
        )
    qc = fe.quant_config
    w13, w13s, w2, w2s = (
        getattr(ex, n)
        for n in ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
    )
    if any(t.untyped_storage().nbytes() == 0 for t in (w13, w13s, w2, w2s)):
        raise MoeWeightError(
            "routed expert storage released (localized / freed weights are not "
            "supported)"
        )
    a1 = qc.a1_gscale.float().reshape(-1)
    if a1.numel() > 1 and not bool((a1 == a1[0]).all()):
        raise MoeWeightError("per-expert routed FC1 input scales differ")
    a2g = _pow2_scalar(qc.a2_gscale, "a2_gscale (FC2 input global scale)")
    g1a = qc.g1_alphas.float().contiguous()
    g1c = fe.g1_scale_c.float()
    if not torch.equal(g1c, (g1a * a2g).float()):
        raise MoeWeightError(
            "g1_scale_c != g1_alphas x a2_gscale; unknown FC2 scale fold"
        )
    beta = fe.gemm1_beta if fe.gemm1_beta is not None else torch.zeros_like(g1a)
    if fe.gemm1_alpha is None or fe.gemm1_clamp_limit is None:
        raise MoeWeightError(
            "SwiGLU-OAI alpha / clamp limit missing (not a swigluoai MoE?)"
        )
    gu_w, gu_s = _linear_mxfp8(
        moe.shared_experts.gate_up_proj, "shared gate_up_proj", (2 * INTER, HIDDEN)
    )
    dn_w, dn_s = _linear_mxfp8(
        moe.shared_experts.down_proj, "shared down_proj", (HIDDEN, INTER)
    )
    w = MoeWeights(
        w13=w13.view(torch.uint8),
        w13_scale=w13s.view(torch.float8_e4m3fn),
        w2=w2.view(torch.uint8),
        w2_scale=w2s.view(torch.float8_e4m3fn),
        g1_alphas=g1a,
        g1_scale_c=g1c.contiguous(),
        g2_alphas=qc.g2_alphas.float().contiguous(),
        act_alpha=fe.gemm1_alpha.float().contiguous(),
        act_beta=beta.float().contiguous(),
        act_clamp=fe.gemm1_clamp_limit.float().contiguous(),
        a1_gscale=a1[:1].contiguous(),
        router=moe.gate.weight.float().contiguous(),
        bias=moe.e_score_correction_bias.float().contiguous(),
        shared_gu_w=gu_w,
        shared_gu_s=gu_s,
        shared_dn_w=dn_w,
        shared_dn_s=dn_s,
        fc2_enc=a2g,
    )
    w.validate()
    return w

prep_mode_q(w)

Mode Q FC2 prep as new tensors (idempotent): fc31_scale_c = g1_alphas x 16, fc2_alpha = w2_scale_2 / 16. Refuses non-placeholder routed FC1 input scales (mode Q's FC1 scale is the post's per-token s_t).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/moe/weights.py
def prep_mode_q(w: MoeWeights) -> MoeWeights:
    """Mode Q FC2 prep as new tensors (idempotent): fc31_scale_c = g1_alphas x 16,
    fc2_alpha = w2_scale_2 / 16.
    Refuses non-placeholder routed FC1 input scales (mode Q's FC1 scale is the post's
    per-token s_t)."""
    if w.fc2_enc == FC2_SCALE_Q:
        return w
    a1 = w.a1_gscale.float().reshape(-1)
    if not bool((a1 == 1.0).all()):
        raise MoeWeightError(
            "mode Q assumes the M3.1 1.0 placeholder FC1 input scale, got "
            f"{a1.tolist()[:4]}"
        )
    if not torch.equal(w.g1_scale_c.float(), (w.g1_alphas.float() * w.fc2_enc).float()):
        raise MoeWeightError(
            "g1_scale_c != g1_alphas x fc2_enc: unknown FC2 scale fold"
        )
    base_g2 = w.g2_alphas.float() * w.fc2_enc  # exact: fc2_enc is a power of two
    return replace(
        w,
        g1_scale_c=(w.g1_alphas.float() * FC2_SCALE_Q).contiguous(),
        g2_alphas=(base_g2 / FC2_SCALE_Q).contiguous(),
        fc2_enc=FC2_SCALE_Q,
    )