Skip to content

vllm.models.mistral.amd.ops.indexer

DSA indexer Q/K epilogues.

Q: SSMax, optional RoPE and per-head UE8M0 FP8 quantization in one launch, with the quantization scale folded into the head weights (extends fused_indexer_q_rope_quant from sparse_attn_indexer). K: fp32 LayerNorm and optional RoPE in one launch, optionally quantized and written straight to the paged FP8 indexer cache. The Q launch can also clear the top-k buffer rows that the indexer fills.

The indexer layout is [rope | nope]. Intermediate roundings match the unfused path.

Functions:

fused_indexer_k_norm_rope(positions, k, weight, bias, eps, cos_sin_cache, is_neox)

Return rope(LayerNorm_fp32(k)) in k.dtype; k is [T, D].

RoPE covers the leading cos_sin_cache.shape[1] dims (skipped if cos_sin_cache is None).

Source code in vllm/models/mistral/amd/ops/indexer.py
def fused_indexer_k_norm_rope(
    positions: torch.Tensor,
    k: torch.Tensor,
    weight: torch.Tensor,
    bias: torch.Tensor,
    eps: float,
    cos_sin_cache: torch.Tensor | None,
    is_neox: bool,
) -> torch.Tensor:
    """Return `rope(LayerNorm_fp32(k))` in `k.dtype`; `k` is `[T, D]`.

    RoPE covers the leading `cos_sin_cache.shape[1]` dims (skipped if
    `cos_sin_cache` is None).
    """
    out = torch.empty_like(k, memory_format=torch.contiguous_format)
    _launch_indexer_k(
        positions, k, weight, bias, eps, cos_sin_cache, is_neox, out, None, None
    )
    return out

fused_indexer_k_norm_rope_cache_write(positions, k, weight, bias, eps, cos_sin_cache, is_neox, kv_cache, slot_mapping, shuffled=None)

Write rope(LayerNorm_fp32(k)) to the paged FP8 indexer cache, as the ROCm insert indexer_k_quant_and_cache_triton does with the bf16 result (UE8M0 scale per 128-dim row; tokens past slot_mapping or with a negative slot are skipped).

shuffled is the cache's value layout, as reported by the cache object (MistralIndexerCache.uses_shuffled_layout); None derives it from the block size with the same rule (indexer_k_cache_is_shuffled).

Source code in vllm/models/mistral/amd/ops/indexer.py
def fused_indexer_k_norm_rope_cache_write(
    positions: torch.Tensor,
    k: torch.Tensor,
    weight: torch.Tensor,
    bias: torch.Tensor,
    eps: float,
    cos_sin_cache: torch.Tensor | None,
    is_neox: bool,
    kv_cache: torch.Tensor,
    slot_mapping: torch.Tensor,
    shuffled: bool | None = None,
) -> None:
    """Write `rope(LayerNorm_fp32(k))` to the paged FP8 indexer cache, as the
    ROCm insert `indexer_k_quant_and_cache_triton` does with the bf16 result
    (UE8M0 scale per 128-dim row; tokens past `slot_mapping` or with a negative
    slot are skipped).

    `shuffled` is the cache's value layout, as reported by the cache object
    (`MistralIndexerCache.uses_shuffled_layout`); None derives it from the
    block size with the same rule (`indexer_k_cache_is_shuffled`)."""
    if shuffled is None:
        shuffled = indexer_k_cache_is_shuffled(kv_cache.shape[1])
    _launch_indexer_k(
        positions,
        k,
        weight,
        bias,
        eps,
        cos_sin_cache,
        is_neox,
        None,
        slot_mapping,
        kv_cache,
        shuffled=shuffled,
    )

fused_indexer_q_ssmax_rope_quant(positions, q, cos_sin_cache, weights, log_positions, s, b, softmax_scale, head_scale, is_neox, topk_indices=None)

Return FP8 q [T, H, D] and fp32 weights * q_scale * scales.

SSMax is applied first, then RoPE on the leading cos_sin_cache.shape[1] dims (skipped if cos_sin_cache is None), then one UE8M0 scale per head. Rows [:T] of topk_indices, if given, are filled with -1.

Source code in vllm/models/mistral/amd/ops/indexer.py
def fused_indexer_q_ssmax_rope_quant(
    positions: torch.Tensor,
    q: torch.Tensor,
    cos_sin_cache: torch.Tensor | None,
    weights: torch.Tensor,
    log_positions: torch.Tensor,
    s: torch.Tensor,
    b: torch.Tensor,
    softmax_scale: float,
    head_scale: float,
    is_neox: bool,
    topk_indices: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Return FP8 `q` `[T, H, D]` and fp32 `weights * q_scale * scales`.

    SSMax is applied first, then RoPE on the leading `cos_sin_cache.shape[1]`
    dims (skipped if `cos_sin_cache` is None), then one UE8M0 scale per head.
    Rows `[:T]` of `topk_indices`, if given, are filled with -1.
    """
    assert current_platform.is_rocm()
    assert q.dtype == torch.bfloat16 and q.stride(-1) == 1
    num_tokens, num_heads, head_dim = q.shape
    assert weights.shape == (num_tokens, num_heads)
    assert log_positions.shape == (num_tokens,)
    assert s.shape == b.shape == (num_heads,)
    rope_dim = 0 if cos_sin_cache is None else cos_sin_cache.shape[1]
    assert 0 <= rope_dim <= head_dim and rope_dim % 2 == 0
    if topk_indices is not None:
        assert topk_indices.dtype == torch.int32 and topk_indices.stride(-1) == 1
        assert topk_indices.shape[0] >= num_tokens
    topk_tokens = 0 if topk_indices is None else topk_indices.shape[1]

    q_fp8 = torch.empty_like(q, dtype=current_platform.fp8_dtype())
    weights_out = torch.empty_like(weights, dtype=torch.float32)
    if num_tokens == 0:
        return q_fp8, weights_out
    fp8_min, fp8_max = get_fp8_min_max()
    _indexer_q_kernel[(num_tokens,)](
        positions,
        q,
        q.stride(0),
        q.stride(1),
        q if cos_sin_cache is None else cos_sin_cache,
        0 if cos_sin_cache is None else cos_sin_cache.stride(0),
        q_fp8,
        q_fp8.stride(0),
        q_fp8.stride(1),
        weights,
        weights.stride(0),
        weights.stride(1),
        weights_out,
        weights_out.stride(0),
        weights_out.stride(1),
        softmax_scale,
        head_scale,
        log_positions,
        s,
        b,
        weights if topk_indices is None else topk_indices,
        0 if topk_indices is None else topk_indices.stride(0),
        fp8_min=fp8_min,
        fp8_max=fp8_max,
        clear_topk=topk_indices is not None,
        topk_tokens=topk_tokens,
        topk_block=triton.next_power_of_2(max(topk_tokens, 1)),
        is_neox=is_neox,
        apply_rope=rope_dim > 0,
        num_heads=num_heads,
        heads_block=triton.next_power_of_2(num_heads),
        head_dim=head_dim,
        dim_block=triton.next_power_of_2(head_dim),
        half_rope_dim=rope_dim // 2,
        rope_block=triton.next_power_of_2(max(rope_dim // 2, 1)),
        nope_block=triton.next_power_of_2(max(head_dim - rope_dim, 1)),
        num_warps=4,
    )
    return q_fp8, weights_out