Skip to content

vllm.model_executor.layers.quantization.utils.humming

Humming quantization integration.

Modules:

  • activation –

    MoE activation expressions for Humming input processing.

  • linear –

    Prepare and execute Humming linear layers.

  • moe –

    Configure, prepare weights for, and assemble Humming MoE kernels.

  • priority –

    Humming kernel selection priority.

  • schema –

    Map Humming schemas and handle shared checkpoint quantization settings.

Functions:

check_and_fallback_input_schema(weight_schema, input_schema, param_dtype=None, allow_fallback=True, allow_fp8_sm100=True, warn=True)

Resolve input_schema against weight_schema, falling back if needed.

Parameters:

  • weight_schema

    (BaseWeightSchema) –

    the weight schema the input must be compatible with.

  • input_schema

    (BaseInputSchema) –

    the requested input schema.

  • param_dtype

    (dtype | None, default: None ) –

    model compute dtype, used as the 16-bit fallback.

  • allow_fallback

    (bool, default: True ) –

    if False, return the requested schema as-is.

  • allow_fp8_sm100

    (bool, default: True ) –

    whether FP8 activations may be honored on SM100-SM119. Humming's dense FP8 UMMA path is fast there, but the MoE grouped-GEMM path is not (the tuner can emit invalid two-CTA configs), so the MoE call sites pass False.

  • warn

    (bool, default: True ) –

    whether to log fallbacks; False when only probing the result.

Source code in vllm/model_executor/layers/quantization/utils/humming/schema.py
def check_and_fallback_input_schema(
    weight_schema: "BaseWeightSchema",
    input_schema: "BaseInputSchema",
    param_dtype: torch.dtype | None = None,
    allow_fallback: bool = True,
    allow_fp8_sm100: bool = True,
    warn: bool = True,
) -> "HummingInputSchema":
    """Resolve *input_schema* against *weight_schema*, falling back if needed.

    Args:
        weight_schema: the weight schema the input must be compatible with.
        input_schema: the requested input schema.
        param_dtype: model compute dtype, used as the 16-bit fallback.
        allow_fallback: if False, return the requested schema as-is.
        allow_fp8_sm100: whether FP8 activations may be honored on
            SM100-SM119. Humming's dense FP8 UMMA path is fast there, but the
            MoE grouped-GEMM path is not (the tuner can emit invalid two-CTA
            configs), so the MoE call sites pass False.
        warn: whether to log fallbacks; False when only probing the result.

    """
    from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
    from vllm.utils.humming import dtypes as humming_dtypes

    allow_fallback = allow_fallback and not envs.VLLM_STRICT_QUANT_SCHEME
    capability = current_platform.get_device_capability()
    assert capability is not None
    sm_version = capability.to_int()
    if param_dtype is None:
        param_dtype = torch.get_default_dtype()
        if param_dtype not in (torch.float16, torch.bfloat16):
            param_dtype = torch.bfloat16 if sm_version >= 80 else torch.float16
    assert param_dtype in (torch.float16, torch.bfloat16)

    dtype_fallback_order_map = {
        humming_dtypes.float8e5m2: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float8e4m3: (
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float8e3m4: (
            humming_dtypes.float8e4m3,
            humming_dtypes.int8,
        ),
        humming_dtypes.float4e2m1: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float4e0m3: (
            humming_dtypes.int4,
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.int8: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
        ),
        humming_dtypes.int4: (
            humming_dtypes.int8,
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
        ),
    }

    weight_schema = weight_schema.to_humming_schema(param_dtype)
    input_schema = input_schema.to_humming_schema(param_dtype)
    fp6_dtypes = (humming_dtypes.float6e2m3, humming_dtypes.float6e3m2)
    if input_schema.a_dtype in fp6_dtypes:
        input_schema.a_dtype = humming_dtypes.float8e4m3
    a_dtype = input_schema.a_dtype
    input_bits = a_dtype.num_bits if a_dtype is not None else 16
    weight_group_size = weight_schema.weight_scale_group_size
    input_group_size = input_schema.input_scale_group_size
    input_scale_dtype = input_schema.input_scale_dtype

    fp8_dtypes = (
        humming_dtypes.float8e4m3,
        humming_dtypes.float8e5m2,
        humming_dtypes.float8e3m4,
    )

    def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
        if dtype is None or dtype.num_bits == 16:
            return False
        is_int4_deprecated = dtype == humming_dtypes.int4 and sm_version >= 90
        is_int8_deprecated = dtype == humming_dtypes.int8 and 103 <= sm_version < 110
        # FP8 is fine when the caller has a fast path for it (dense UMMA):
        # other sub-16-bit dtypes (e.g. FP4) still lack fast kernels on
        # SM10X/SM11X.
        is_low_bit_deprecated = (
            dtype.num_bits < 16
            and (dtype not in fp8_dtypes or not allow_fp8_sm100)
            and 100 <= sm_version < 120
        )
        return is_int4_deprecated or is_int8_deprecated or is_low_bit_deprecated

    if input_schema.is_compatible_with(weight_schema, param_dtype):
        if not allow_fallback:
            if warn and is_deprecated(a_dtype):
                msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
                logger.warning_once(msg)
            return input_schema

        if a_dtype is None or a_dtype.num_bits == 16:
            return input_schema

        if not is_deprecated(a_dtype):
            is_mxfp8 = (
                a_dtype.is_floating_point_type
                and a_dtype.num_bits == 8
                and input_group_size == 32
                and input_scale_dtype == humming_dtypes.float8e8m0
                and sm_version >= (100 if allow_fp8_sm100 else 120)
            )
            is_groupwise = input_group_size > 0 or weight_group_size > 0
            if input_bits == 8 and is_groupwise and not is_mxfp8:
                # prefer wna16 for groupwise weight or groupwise input
                if warn:
                    _warn_input_fallback(a_dtype, None)
                return HummingInputSchema()

            return input_schema

    if allow_fallback:
        fallback_dtypes = (
            a_dtype,
            *dtype_fallback_order_map.get(a_dtype, ()),
            humming_dtypes.DataType.from_any(param_dtype),
        )
        for dtype in fallback_dtypes:
            if dtype is None or is_deprecated(dtype):
                continue

            group_size = 0
            scale_dtype = humming_dtypes.float32
            quant_mode = InputQuantizationMode.DynamicToken
            if dtype.num_bits == 16:
                scale_dtype = None
                quant_mode = InputQuantizationMode.Disabled
            elif dtype in (humming_dtypes.float8e4m3, humming_dtypes.float8e3m4):
                is_channelwise_weight = weight_schema.weight_scale_group_size == 0
                weight_group_size = weight_schema.weight_scale_group_size
                is_e8m0_scale = weight_schema.bs_dtype == humming_dtypes.float8e8m0
                is_mx_weight = weight_group_size == 32 and is_e8m0_scale
                if sm_version >= 120 and (is_channelwise_weight or is_mx_weight):
                    group_size = 32
                    scale_dtype = humming_dtypes.float8e8m0
                    quant_mode = InputQuantizationMode.DynamicGroup

            candidate = HummingInputSchema(
                a_dtype=dtype,
                input_scale_group_size=group_size,
                input_scale_dtype=scale_dtype,
                input_quant_mode=quant_mode,
            )
            if candidate.is_compatible_with(weight_schema, param_dtype):
                if warn:
                    _warn_input_fallback(a_dtype, dtype)
                return candidate

    raise ValueError(
        f"No compatible Humming input schema for {input_schema} with "
        f"weight schema {weight_schema} on SM{sm_version} "
        f"({allow_fallback=})"
    )

convert_linear_layer_to_humming_standard(layer, name_map)

Rename/reshape a linear layer's quantized params (the canonical MPLinear layout: weight_packed int32 + weight_scale) into the parameter names and layout humming's weight schema expects (weight / weight_scale).

Source code in vllm/model_executor/layers/quantization/utils/humming/linear.py
def convert_linear_layer_to_humming_standard(
    layer: LinearBase, name_map: dict[str, str]
):
    """Rename/reshape a linear layer's quantized params (the canonical MPLinear
    layout: `weight_packed` int32 + `weight_scale`) into the parameter names
    and layout humming's weight schema expects (`weight` / `weight_scale`)."""
    for name, checkpoint_name in name_map.items():
        param = getattr(layer, checkpoint_name)
        tensor = param.data

        if name == "weight":
            # (input_dim, output_dim) after PWAL, if it transposed the weight.
            input_dim, output_dim = getattr(
                layer,
                "processed_weight_dims",
                (getattr(param, "input_dim", 1), getattr(param, "output_dim", 0)),
            )

            if input_dim == 0 and output_dim == 1:
                tensor = tensor.transpose(1, 0).contiguous()
            else:
                assert output_dim == 0 and input_dim == 1

            tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
        elif name in ["weight_scale", "zero_point"]:
            if getattr(param, "output_dim", 0) == 1:
                tensor = tensor.transpose(0, 1).contiguous()
            if tensor.ndim == 1:
                tensor = tensor.unsqueeze(1)

            tensor = tensor.view(torch.int32) if name == "zero_point" else tensor

        if name != checkpoint_name:
            # One param under both names: the loader keeps its name.
            move_parameter(layer, checkpoint_name, name)
        replace_parameter(layer, name, tensor)

convert_to_humming_moe_kernel_format(layer, quant_config=None, sublayer_configs=None, weight_schema=None, input_schema=None, force_weight_schema=None, allow_input_schema_fallback=True)

Convert MoE weights from checkpoint format to Humming kernel format.

This function processes weights for each sublayer (w13, w2) by: 1. Converting from checkpoint format to humming format if needed 2. Force requanting if a different quantization schema is specified 3. Preparing layer metadata for the Humming kernel 4. Transforming weights for inference

Parameters:

  • layer

    (RoutedExperts) –

    The RoutedExperts layer containing weights to process

  • quant_config

    (dict | None, default: None ) –

    Optional quantization config dict. Required if weight_schema or input_schema are None. Used to build schemas via BaseWeightSchema.from_config().

  • sublayer_configs

    (dict[str, Any] | None, default: None ) –

    Optional configuration dict for each sublayer (w13, w2). Each config must have "shape_n" and "shape_k" keys. If None, configs are built from layer.moe_config properties.

  • weight_schema

    (Any | None, default: None ) –

    Optional initial weight quantization schema. If None, built from quant_config.

  • input_schema

    (Any | None, default: None ) –

    Optional initial input quantization schema. If None, built from quant_config or env vars.

  • force_weight_schema

    (Any | None, default: None ) –

    Optional schema to force requantization to

  • allow_input_schema_fallback

    (bool, default: True ) –

    Whether incompatible input schemas may be replaced.

Side effects
  • Modifies layer parameters in place
  • Sets layer.weight_schemas and layer.input_schemas
  • Sets layer.humming_configs for quant config construction
Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py
def convert_to_humming_moe_kernel_format(
    layer: "RoutedExperts",
    quant_config: dict | None = None,
    sublayer_configs: dict[str, Any] | None = None,
    weight_schema: Any | None = None,
    input_schema: Any | None = None,
    force_weight_schema: Any | None = None,
    allow_input_schema_fallback: bool = True,
) -> dict[str, "LayerConfig"]:
    """Convert MoE weights from checkpoint format to Humming kernel format.

    This function processes weights for each sublayer (w13, w2) by:
    1. Converting from checkpoint format to humming format if needed
    2. Force requanting if a different quantization schema is specified
    3. Preparing layer metadata for the Humming kernel
    4. Transforming weights for inference

    Args:
        layer: The RoutedExperts layer containing weights to process
        quant_config: Optional quantization config dict. Required if weight_schema
                     or input_schema are None. Used to build schemas via
                     BaseWeightSchema.from_config().
        sublayer_configs: Optional configuration dict for each sublayer (w13, w2).
                         Each config must have "shape_n" and "shape_k" keys.
                         If None, configs are built from layer.moe_config properties.
        weight_schema: Optional initial weight quantization schema.
                      If None, built from quant_config.
        input_schema: Optional initial input quantization schema.
                     If None, built from quant_config or env vars.
        force_weight_schema: Optional schema to force requantization to
        allow_input_schema_fallback: Whether incompatible input schemas may be replaced.

    Side effects:
        - Modifies layer parameters in place
        - Sets layer.weight_schemas and layer.input_schemas
        - Sets layer.humming_configs for quant config construction

    """
    # Build schemas from quant_config if not provided
    has_bias = layer.moe_config.has_bias
    num_experts = layer.moe_config.num_local_experts
    param_dtype = layer.params_dtype

    if weight_schema is None or input_schema is None:
        if quant_config is None:
            raise ValueError(
                "Must provide either weight_schema/input_schema or quant_config"
            )

        from vllm.utils.humming import BaseWeightSchema, HummingInputSchema

        if weight_schema is None:
            weight_schema = BaseWeightSchema.from_config(quant_config)

        if input_schema is None:
            input_quant_config = (envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}).copy()
            if humming_is_layer_skipped(input_quant_config, layer.layer_name):
                input_schema = HummingInputSchema()
            else:
                # TODO: read input_quant_config from quant_config
                input_quant_config = humming_schema.resolve_humming_layer_config(
                    input_quant_config, layer.layer_name
                )
                allow_input_schema_fallback = input_quant_config.pop(
                    "allow_fallback", False
                )
                input_schema = HummingInputSchema.from_config(input_quant_config)

    # Build sublayer configs from layer properties if not provided
    if sublayer_configs is None:
        is_gated = layer.moe_config.activation.is_gated
        intermediate_size = layer.moe_config.intermediate_size_per_partition
        sublayer_configs = {
            "w13": {
                "shape_n": intermediate_size * (2 if is_gated else 1),
                "shape_k": layer.moe_config.hidden_dim,
            },
            "w2": {
                "shape_n": layer.moe_config.hidden_dim,
                "shape_k": intermediate_size,
            },
        }

    layer.weight_schemas = {}
    layer.input_schemas = {}
    humming_configs = {}

    for sublayer_name, configs in sublayer_configs.items():
        final_weight_schema, final_input_schema, humming_config = (
            _process_single_sublayer(
                layer=layer,
                sublayer_name=sublayer_name,
                shape_n=configs["shape_n"],
                shape_k=configs["shape_k"],
                weight_schema=weight_schema,
                input_schema=input_schema,
                has_bias=has_bias,
                num_experts=num_experts,
                param_dtype=param_dtype,
                force_weight_schema=force_weight_schema,
                allow_input_schema_fallback=allow_input_schema_fallback,
            )
        )

        layer.weight_schemas[sublayer_name] = final_weight_schema
        layer.input_schemas[sublayer_name] = final_input_schema
        humming_configs[sublayer_name] = humming_config

    layer.humming_configs = humming_configs
    return humming_configs

humming_executed_input_quant_key(weight_quant_config, activation_key, allow_fallback=True)

The activation quantization Humming runs on this device for weights described by weight_quant_config when activation_key is requested. None if it runs unquantized activations or rejects the request.

Source code in vllm/model_executor/layers/quantization/utils/humming/schema.py
def humming_executed_input_quant_key(
    weight_quant_config: dict[str, Any],
    activation_key: QuantKey | None,
    allow_fallback: bool = True,
) -> QuantKey | None:
    """The activation quantization Humming runs on this device for weights
    described by `weight_quant_config` when `activation_key` is requested.
    None if it runs unquantized activations or rejects the request."""
    from vllm.utils.humming import BaseWeightSchema

    if activation_key is None:
        return None
    try:
        executed = check_and_fallback_input_schema(
            BaseWeightSchema.from_config(weight_quant_config),
            quant_key_to_input_schema(activation_key),
            allow_fallback=allow_fallback,
            warn=False,
        )
    except ValueError:
        return None
    return input_schema_to_quant_key(executed)

prefers_humming(compute_capability=None)

Whether Humming outranks Marlin. A missing compute capability uses the current device.

Source code in vllm/model_executor/layers/quantization/utils/humming/priority.py
def prefers_humming(compute_capability: int | None = None) -> bool:
    """Whether Humming outranks Marlin. A missing compute capability uses the
    current device."""
    if compute_capability is None:
        from vllm.platforms import current_platform

        if current_platform.is_cuda():
            cc = current_platform.get_device_capability()
            compute_capability = cc.to_int() if cc is not None else None
    return compute_capability in _HUMMING_PREFERRED_CAPABILITIES

prioritize_humming(kernels, compute_capability=None)

Move Humming directly ahead of Marlin where Humming is preferred.

Every other entry keeps its relative order and the input list is not modified. Match kernel class names or MoE backend enum names.

Source code in vllm/model_executor/layers/quantization/utils/humming/priority.py
def prioritize_humming(
    kernels: list[_KernelT],
    compute_capability: int | None = None,
) -> list[_KernelT]:
    """Move Humming directly ahead of Marlin where Humming is preferred.

    Every other entry keeps its relative order and the input list is not
    modified. Match kernel class names or MoE backend enum names.
    """
    if not prefers_humming(compute_capability):
        return kernels

    names = [
        (kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
        for kernel in kernels
    ]
    humming = next((i for i, name in enumerate(names) if "humming" in name), None)
    marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
    if humming is None or marlin is None or humming < marlin:
        return kernels
    kernels = kernels.copy()
    kernels.insert(marlin, kernels.pop(humming))
    return kernels

select_humming_moe_experts(config, weight_key, activation_key)

Select the primary Humming MoE Experts class Note: Shape-specific fallbacks may still occur at runtime.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py
def select_humming_moe_experts(
    config: FusedMoEConfig,
    weight_key: QuantKey | None,
    activation_key: QuantKey | None,
) -> type[mk.FusedMoEExperts] | None:
    """Select the primary Humming MoE Experts class
    Note: Shape-specific fallbacks may still occur at runtime.
    """
    if not has_humming():
        return None

    # NOTE: the kernels are selected in the following order.
    AVAILABLE_EXPERTS: list[type[mk.FusedMoEExperts]] = [
        BatchedHummingGroupedExperts,
        HummingGroupedExperts,
        HummingIndexedExperts,
    ]

    # NOTE(rob): We need to peak into the P/F selection to determine
    # if we are using the batched or standard expert format, which
    # if not ideal. Once we unify TP + DP/EP, we can select P/F first.
    activation_format = config.activation_format

    def _make_log_backend(experts_cls: type[mk.FusedMoEExperts]):
        return f"Using {experts_cls.__name__} Humming MoE backend."

    def _make_log_unsupported(
        experts_cls: type[mk.FusedMoEExperts], reason: str | None
    ) -> str:
        if reason:
            return (
                f"Humming MoE experts {experts_cls.__name__} does not support the "
                f"deployment configuration since {reason}."
            )
        else:
            return (
                f"Humming MoE experts '{experts_cls.__name__}' does not support the "
                "deployment configuration."
            )

    for k_cls in AVAILABLE_EXPERTS:
        supported, reason = k_cls.is_supported_config(
            k_cls,
            config,
            weight_key,
            activation_key,
            activation_format,
        )
        if supported:
            logger.info_once(_make_log_backend(k_cls))
            return k_cls
        else:
            logger.debug_once(_make_log_unsupported(k_cls, reason))

    return None

stage_moe_weights_for_humming(layer, w13, w2, w13_scale, w2_scale, w13_input_scale=None, w2_input_scale=None)

Install canonical weights and scales for Humming's layer-based converter.

Returns the scale parameter name used, weight_scale or weight_scale_inv.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py
def stage_moe_weights_for_humming(
    layer: "RoutedExperts",
    w13: torch.Tensor,
    w2: torch.Tensor,
    w13_scale: torch.Tensor,
    w2_scale: torch.Tensor,
    w13_input_scale: torch.Tensor | None = None,
    w2_input_scale: torch.Tensor | None = None,
) -> str:
    """Install canonical weights and scales for Humming's layer-based converter.

    Returns the scale parameter name used, `weight_scale` or
    `weight_scale_inv`.
    """
    scale_name = (
        "weight_scale_inv" if hasattr(layer, "w13_weight_scale_inv") else "weight_scale"
    )
    replace_parameter(layer, "w13_weight", w13)
    replace_parameter(layer, "w2_weight", w2)
    replace_parameter(layer, f"w13_{scale_name}", w13_scale)
    replace_parameter(layer, f"w2_{scale_name}", w2_scale)
    if w13_input_scale is not None:
        replace_parameter(layer, "w13_input_scale", w13_input_scale)
    if w2_input_scale is not None:
        replace_parameter(layer, "w2_input_scale", w2_input_scale)
    return scale_name