Skip to content

vllm.kernels.triton.fused_pre_fc_norms

Functions:

fused_pre_fc_norms(inputs_embeds, hidden_states, emb_weight, hid_weight, eps)

Returns cat([norm_e(inputs_embeds), norm_h(hidden_states)]) -> [N, 2H].

Both norms are Gemma-style RMSNorm (scale by 1 + weight, fp32 accumulation, multiply in fp32 before the cast back to the input dtype).

Source code in vllm/kernels/triton/fused_pre_fc_norms.py
def fused_pre_fc_norms(
    inputs_embeds: torch.Tensor,
    hidden_states: torch.Tensor,
    emb_weight: torch.Tensor,
    hid_weight: torch.Tensor,
    eps: float,
) -> torch.Tensor:
    """Returns cat([norm_e(inputs_embeds), norm_h(hidden_states)]) -> [N, 2H].

    Both norms are Gemma-style RMSNorm (scale by `1 + weight`, fp32
    accumulation, multiply in fp32 before the cast back to the input dtype).
    """
    n, h = inputs_embeds.shape
    out = torch.empty(n, 2 * h, dtype=inputs_embeds.dtype, device=inputs_embeds.device)
    _fused_pre_fc_norms_kernel[(n,)](
        inputs_embeds,
        inputs_embeds.stride(0),
        hidden_states,
        hidden_states.stride(0),
        emb_weight,
        hid_weight,
        eps,
        out,
        out.stride(0),
        h,
        triton.next_power_of_2(h),
    )
    return out

fused_pre_fc_norms_split(inputs_embeds, hidden_states, emb_weight, hid_weight, eps)

Two-width variant of fused_pre_fc_norms with separate outputs.

For drafters whose pre-FC norms have different widths and feed separate FC GEMMs (e.g. Qwen4Exp MTP), so one launch writes two contiguous outputs instead of a concat. Returns (normed_embeds [N, H_EMB], normed_hidden [N, H_HID]).

Source code in vllm/kernels/triton/fused_pre_fc_norms.py
def fused_pre_fc_norms_split(
    inputs_embeds: torch.Tensor,
    hidden_states: torch.Tensor,
    emb_weight: torch.Tensor,
    hid_weight: torch.Tensor,
    eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Two-width variant of `fused_pre_fc_norms` with separate outputs.

    For drafters whose pre-FC norms have different widths and feed separate
    FC GEMMs (e.g. Qwen4Exp MTP), so one launch writes two contiguous
    outputs instead of a concat. Returns `(normed_embeds [N, H_EMB],
    normed_hidden [N, H_HID])`.
    """
    assert inputs_embeds.dim() == 2 and inputs_embeds.stride(1) == 1
    assert hidden_states.dim() == 2 and hidden_states.stride(1) == 1
    assert emb_weight.is_contiguous() and hid_weight.is_contiguous()

    n, h_emb = inputs_embeds.shape
    h_hid = hidden_states.shape[1]
    emb_out = torch.empty(
        n, h_emb, dtype=inputs_embeds.dtype, device=inputs_embeds.device
    )
    hid_out = torch.empty(
        n, h_hid, dtype=hidden_states.dtype, device=hidden_states.device
    )
    _fused_pre_fc_norms_split_kernel[(n,)](
        inputs_embeds,
        inputs_embeds.stride(0),
        hidden_states,
        hidden_states.stride(0),
        emb_weight,
        hid_weight,
        emb_out,
        emb_out.stride(0),
        hid_out,
        hid_out.stride(0),
        eps,
        H_EMB=h_emb,
        H_HID=h_hid,
        launch_pdl=current_platform.is_arch_support_pdl(),
        num_warps=max(4, min(32, triton.next_power_of_2(h_hid) // 1024)),
    )
    return emb_out, hid_out