Skip to content

vllm.model_executor.layers.quantization.utils.humming.schema

Map Humming schemas and handle shared checkpoint quantization settings.

Functions:

_group_shape(group_size, group_size_n=0)

Map humming group sizes to QuantKey GroupShape.

group_size: elements per group along K (col); 0 means full dimension. group_size_n: elements per group along N (row); 0 means 1 (per-row).

GroupShape convention: row = N dim, col = K dim.

Source code in vllm/model_executor/layers/quantization/utils/humming/schema.py
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
    """Map humming group sizes to QuantKey GroupShape.

    group_size:   elements per group along K (col); 0 means full dimension.
    group_size_n: elements per group along N (row); 0 means 1 (per-row).

    GroupShape convention: row = N dim, col = K dim.
    """
    if group_size == 0 and group_size_n == 0:
        return GroupShape.PER_CHANNEL

    row = group_size_n if group_size_n > 0 else 1
    col = group_size if group_size > 0 else -1
    return GroupShape(row=row, col=col)

check_and_fallback_input_schema(weight_schema, input_schema, param_dtype=None, allow_fallback=True, allow_fp8_sm100=True, warn=True)

Resolve input_schema against weight_schema, falling back if needed.

Parameters:

  • weight_schema

    (BaseWeightSchema) –

    the weight schema the input must be compatible with.

  • input_schema

    (BaseInputSchema) –

    the requested input schema.

  • param_dtype

    (dtype | None, default: None ) –

    model compute dtype, used as the 16-bit fallback.

  • allow_fallback

    (bool, default: True ) –

    if False, return the requested schema as-is.

  • allow_fp8_sm100

    (bool, default: True ) –

    whether FP8 activations may be honored on SM100-SM119. Humming's dense FP8 UMMA path is fast there, but the MoE grouped-GEMM path is not (the tuner can emit invalid two-CTA configs), so the MoE call sites pass False.

  • warn

    (bool, default: True ) –

    whether to log fallbacks; False when only probing the result.

Source code in vllm/model_executor/layers/quantization/utils/humming/schema.py
def check_and_fallback_input_schema(
    weight_schema: "BaseWeightSchema",
    input_schema: "BaseInputSchema",
    param_dtype: torch.dtype | None = None,
    allow_fallback: bool = True,
    allow_fp8_sm100: bool = True,
    warn: bool = True,
) -> "HummingInputSchema":
    """Resolve *input_schema* against *weight_schema*, falling back if needed.

    Args:
        weight_schema: the weight schema the input must be compatible with.
        input_schema: the requested input schema.
        param_dtype: model compute dtype, used as the 16-bit fallback.
        allow_fallback: if False, return the requested schema as-is.
        allow_fp8_sm100: whether FP8 activations may be honored on
            SM100-SM119. Humming's dense FP8 UMMA path is fast there, but the
            MoE grouped-GEMM path is not (the tuner can emit invalid two-CTA
            configs), so the MoE call sites pass False.
        warn: whether to log fallbacks; False when only probing the result.

    """
    from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
    from vllm.utils.humming import dtypes as humming_dtypes

    allow_fallback = allow_fallback and not envs.VLLM_STRICT_QUANT_SCHEME
    capability = current_platform.get_device_capability()
    assert capability is not None
    sm_version = capability.to_int()
    if param_dtype is None:
        param_dtype = torch.get_default_dtype()
        if param_dtype not in (torch.float16, torch.bfloat16):
            param_dtype = torch.bfloat16 if sm_version >= 80 else torch.float16
    assert param_dtype in (torch.float16, torch.bfloat16)

    dtype_fallback_order_map = {
        humming_dtypes.float8e5m2: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float8e4m3: (
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float8e3m4: (
            humming_dtypes.float8e4m3,
            humming_dtypes.int8,
        ),
        humming_dtypes.float4e2m1: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.float4e0m3: (
            humming_dtypes.int4,
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
            humming_dtypes.int8,
        ),
        humming_dtypes.int8: (
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
        ),
        humming_dtypes.int4: (
            humming_dtypes.int8,
            humming_dtypes.float8e4m3,
            humming_dtypes.float8e3m4,
        ),
    }

    weight_schema = weight_schema.to_humming_schema(param_dtype)
    input_schema = input_schema.to_humming_schema(param_dtype)
    fp6_dtypes = (humming_dtypes.float6e2m3, humming_dtypes.float6e3m2)
    if input_schema.a_dtype in fp6_dtypes:
        input_schema.a_dtype = humming_dtypes.float8e4m3
    a_dtype = input_schema.a_dtype
    input_bits = a_dtype.num_bits if a_dtype is not None else 16
    weight_group_size = weight_schema.weight_scale_group_size
    input_group_size = input_schema.input_scale_group_size
    input_scale_dtype = input_schema.input_scale_dtype

    fp8_dtypes = (
        humming_dtypes.float8e4m3,
        humming_dtypes.float8e5m2,
        humming_dtypes.float8e3m4,
    )

    def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
        if dtype is None or dtype.num_bits == 16:
            return False
        is_int4_deprecated = dtype == humming_dtypes.int4 and sm_version >= 90
        is_int8_deprecated = dtype == humming_dtypes.int8 and 103 <= sm_version < 110
        # FP8 is fine when the caller has a fast path for it (dense UMMA):
        # other sub-16-bit dtypes (e.g. FP4) still lack fast kernels on
        # SM10X/SM11X.
        is_low_bit_deprecated = (
            dtype.num_bits < 16
            and (dtype not in fp8_dtypes or not allow_fp8_sm100)
            and 100 <= sm_version < 120
        )
        return is_int4_deprecated or is_int8_deprecated or is_low_bit_deprecated

    if input_schema.is_compatible_with(weight_schema, param_dtype):
        if not allow_fallback:
            if warn and is_deprecated(a_dtype):
                msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
                logger.warning_once(msg)
            return input_schema

        if a_dtype is None or a_dtype.num_bits == 16:
            return input_schema

        if not is_deprecated(a_dtype):
            is_mxfp8 = (
                a_dtype.is_floating_point_type
                and a_dtype.num_bits == 8
                and input_group_size == 32
                and input_scale_dtype == humming_dtypes.float8e8m0
                and sm_version >= (100 if allow_fp8_sm100 else 120)
            )
            is_groupwise = input_group_size > 0 or weight_group_size > 0
            if input_bits == 8 and is_groupwise and not is_mxfp8:
                # prefer wna16 for groupwise weight or groupwise input
                if warn:
                    _warn_input_fallback(a_dtype, None)
                return HummingInputSchema()

            return input_schema

    if allow_fallback:
        fallback_dtypes = (
            a_dtype,
            *dtype_fallback_order_map.get(a_dtype, ()),
            humming_dtypes.DataType.from_any(param_dtype),
        )
        for dtype in fallback_dtypes:
            if dtype is None or is_deprecated(dtype):
                continue

            group_size = 0
            scale_dtype = humming_dtypes.float32
            quant_mode = InputQuantizationMode.DynamicToken
            if dtype.num_bits == 16:
                scale_dtype = None
                quant_mode = InputQuantizationMode.Disabled
            elif dtype in (humming_dtypes.float8e4m3, humming_dtypes.float8e3m4):
                is_channelwise_weight = weight_schema.weight_scale_group_size == 0
                weight_group_size = weight_schema.weight_scale_group_size
                is_e8m0_scale = weight_schema.bs_dtype == humming_dtypes.float8e8m0
                is_mx_weight = weight_group_size == 32 and is_e8m0_scale
                if sm_version >= 120 and (is_channelwise_weight or is_mx_weight):
                    group_size = 32
                    scale_dtype = humming_dtypes.float8e8m0
                    quant_mode = InputQuantizationMode.DynamicGroup

            candidate = HummingInputSchema(
                a_dtype=dtype,
                input_scale_group_size=group_size,
                input_scale_dtype=scale_dtype,
                input_quant_mode=quant_mode,
            )
            if candidate.is_compatible_with(weight_schema, param_dtype):
                if warn:
                    _warn_input_fallback(a_dtype, dtype)
                return candidate

    raise ValueError(
        f"No compatible Humming input schema for {input_schema} with "
        f"weight schema {weight_schema} on SM{sm_version} "
        f"({allow_fallback=})"
    )

humming_executed_input_quant_key(weight_quant_config, activation_key, allow_fallback=True)

The activation quantization Humming runs on this device for weights described by weight_quant_config when activation_key is requested. None if it runs unquantized activations or rejects the request.

Source code in vllm/model_executor/layers/quantization/utils/humming/schema.py
def humming_executed_input_quant_key(
    weight_quant_config: dict[str, Any],
    activation_key: QuantKey | None,
    allow_fallback: bool = True,
) -> QuantKey | None:
    """The activation quantization Humming runs on this device for weights
    described by `weight_quant_config` when `activation_key` is requested.
    None if it runs unquantized activations or rejects the request."""
    from vllm.utils.humming import BaseWeightSchema

    if activation_key is None:
        return None
    try:
        executed = check_and_fallback_input_schema(
            BaseWeightSchema.from_config(weight_quant_config),
            quant_key_to_input_schema(activation_key),
            allow_fallback=allow_fallback,
            warn=False,
        )
    except ValueError:
        return None
    return input_schema_to_quant_key(executed)