Skip to content

vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse

Classes:

Functions:

ROCMAiterMLASparseImpl

Bases: SparseMLACommonImpl[ROCMAiterMLASparseMetadata]

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
    supports_dcp = True
    can_return_lse_for_decode = True
    use_aiter_sparse_mla = False

    def __init__(
        self,
        num_heads: int,
        head_size: int,
        scale: float,
        num_kv_heads: int,
        alibi_slopes: list[float] | None,
        sliding_window: int | None,
        kv_cache_dtype: str,
        logits_soft_cap: float | None,
        attn_type: str,
        kv_sharing_target_layer_name: str | None,
        # MLA Specific Arguments
        topk_indices_buffer: torch.Tensor | None = None,
        indexer: "Indexer | None" = None,
        **mla_args,
    ) -> None:
        AiterMLAHelper.check_num_heads_validity(num_heads)

        sinks = mla_args.pop("sinks", None)
        if sinks is not None:
            if sinks.dtype != torch.float32:
                raise ValueError(
                    f"ROCm AITER MLA sinks must be float32, got {sinks.dtype}"
                )
            if sinks.ndim != 1 or sinks.numel() != num_heads:
                raise ValueError(
                    "ROCm AITER MLA sinks must have shape "
                    f"({num_heads},), got {tuple(sinks.shape)}"
                )
            if not sinks.is_contiguous():
                raise ValueError("ROCm AITER MLA sinks must be contiguous")
        self.sinks: torch.Tensor | None = sinks
        super().__init__(
            num_heads,
            head_size,
            scale,
            num_kv_heads,
            alibi_slopes,
            sliding_window,
            kv_cache_dtype,
            logits_soft_cap,
            attn_type,
            kv_sharing_target_layer_name,
            indexer=indexer,
            topk_indices_buffer=topk_indices_buffer,
            **mla_args,
        )
        self.softmax_scale = scale
        if self.dcp_world_size > 1 and (
            self.kv_cache_dtype != "fp8_ds_mla" or sinks is not None
        ):
            raise NotImplementedError(
                "ROCm sparse DCP requires fp8_ds_mla without sinks"
            )
        if self.kv_cache_dtype == "fp8_ds_mla":
            # The fp8_ds_mla branch of forward_mqa routes to aiter's Triton
            # sparse_mla_fwd, which is validated on gfx950 only.
            from vllm.platforms.rocm import on_gfx950

            if not on_gfx950():
                raise NotImplementedError(
                    "ROCM_AITER_MLA_SPARSE with kv_cache_dtype=fp8_ds_mla "
                    "requires gfx950 (aiter Triton sparse_mla_fwd, "
                    "fp8_dsv32_mla layout)."
                )
        # Dense-MHA prefill gathers cache rows into a bf16 workspace and would
        # misread packed mxfp4_mla rows.
        self.supports_dense_mha_prefill = (  # type: ignore[misc]
            sinks is None and kv_cache_dtype != "mxfp4_mla"
        )
        self.masked_mha_available = False

        vllm_config = get_current_vllm_config()
        self.owns_indexer: bool = indexer is not None
        self.dcp_scores = getattr(indexer, "dcp_scores", None)
        self.index_epoch: int = _index_epoch(
            self.topk_indices_buffer, self.owns_indexer
        )
        max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
        q_concat_shape = (max_tokens, num_heads, head_size)
        (self.q_concat_buffer,) = current_workspace_manager().get_simultaneous(
            (q_concat_shape, vllm_config.model_config.dtype),
        )
        self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"]
        rank = self.kv_lora_rank
        if kv_cache_dtype == "mxfp4_mla" and (rank < 128 or rank & (rank - 1)):
            # The read loads each packed row as int32 words into one
            # power-of-two tile.
            raise ValueError(
                f"mxfp4_mla needs kv_lora_rank to be a power of two >= 128, got {rank}"
            )
        if kv_cache_dtype == "mxfp4_mla" and self.qk_rope_head_dim != 0:
            raise ValueError(
                "mxfp4_mla stores only the latent, so it needs a rope-free (NoPE) "
                f"model; got qk_rope_head_dim={self.qk_rope_head_dim}"
            )

        if rocm_aiter_ops.is_triton_sparse_mla_enabled():
            reason = self._aiter_sparse_mla_unsupported_reason(vllm_config)
            if reason is None:
                self.use_aiter_sparse_mla = True
            else:
                logger.warning_once(
                    "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is set, but %s; "
                    "using the default sparse MLA path instead.",
                    reason,
                )

    def _aiter_sparse_mla_unsupported_reason(
        self, vllm_config: VllmConfig
    ) -> str | None:
        model_dtype = vllm_config.model_config.dtype
        if model_dtype != torch.bfloat16:
            return f"the model dtype is {model_dtype}, not bfloat16"
        cache_dtype = self.kv_cache_dtype
        if cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
            return f"kv-cache dtype {cache_dtype!r} is neither bfloat16 nor e4m3 fp8"
        # Context parallelism merges each rank's partials with a per-token LSE.
        if self.dcp_world_size > 1 or self.pcp_world_size > 1:
            return "context parallelism is not supported"
        return None

    def do_kv_cache_update(
        self,
        kv_c_normed: torch.Tensor,
        k_pe: torch.Tensor,
        kv_cache: torch.Tensor,
        slot_mapping: torch.Tensor,
        kv_cache_dtype: str,
        k_scale: torch.Tensor,
        skip_scale: bool = False,
    ) -> None:
        if kv_cache_dtype != "mxfp4_mla":
            return super().do_kv_cache_update(
                kv_c_normed,
                k_pe,
                kv_cache,
                slot_mapping,
                kv_cache_dtype,
                k_scale,
                skip_scale,
            )
        if kv_cache.numel() == 0:
            return
        assert k_pe.numel() == 0, (
            f"mxfp4_mla expects a rope-free latent, got k_pe with {k_pe.numel()} "
            "elements"
        )
        store_mxfp4_mla(kv_c_normed, slot_mapping.flatten(), kv_cache)

    def record_logical_topk_ready(self) -> None:
        # This impl shares the top-k indices buffer via SharedTopkIndicesBuffer
        # but does not participate in sparse-MLA index groups.
        pass

    def _forward_mla(
        self,
        layer: AttentionLayer,
        q: torch.Tensor,  # [sq, heads, d_qk]
        kv_c_and_k_pe_cache: torch.Tensor,  # [blocks, heads, d_qk]
        attn_metadata: ROCMAiterMLASparseMetadata,
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        num_tokens = q.shape[0]
        base_mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self.num_heads)
        mla_num_heads = base_mla_num_heads
        need_lse = self.sinks is not None
        from vllm.platforms.rocm import on_gfx942

        # Keep sink attention available for dtypes/head shapes without an
        # AITER return-LSE kernel. Sink layers never need persistent metadata.
        triton_sink_fallback = (
            need_lse
            and q.dtype == kv_c_and_k_pe_cache.dtype
            and (
                q.dtype == torch.float16
                or (
                    q.dtype == torch.bfloat16
                    and (
                        mla_num_heads > 128
                        or (on_gfx942() and 32 < mla_num_heads <= 64)
                    )
                )
            )
        )

        # Sink attention has its own Triton/LSE dispatch below.
        triton_prefill_fallback = not need_lse and _use_triton_long_bf16_prefill(
            kv_c_and_k_pe_cache.dtype, attn_metadata
        )
        if (
            triton_sink_fallback
            or triton_prefill_fallback
            or _use_rocm_sparse_triton(
                kv_cache_dtype=self.kv_cache_dtype,
                head_size=q.shape[-1],
                kv_lora_rank=self.kv_lora_rank,
                num_prefills=attn_metadata.num_prefills,
                num_decodes=attn_metadata.num_decodes,
                num_decode_tokens=attn_metadata.num_decode_tokens,
                max_query_len=attn_metadata.max_query_len,
            )
        ):
            output = torch.empty(
                [num_tokens, q.shape[1], self.kv_lora_rank],
                dtype=attn_metadata.attn_out_dtype,
                device=q.device,
            )
            triton_sinks = None
            if self.sinks is not None:
                triton_sinks = AiterMLAHelper.get_mla_padded_q(
                    self.num_heads,
                    self.sinks.reshape(1, self.num_heads, 1),
                    q.shape[1],
                ).reshape(-1)
            is_mxfp4 = self.kv_cache_dtype == "mxfp4_mla"
            row_width = mxfp4_row_bytes(self.kv_lora_rank) if is_mxfp4 else q.shape[-1]
            kv = kv_c_and_k_pe_cache.view(-1, 1, row_width)
            decode_num_splits = (
                rocm_sparse_decode_bf16_num_splits(
                    num_tokens,
                    q.shape[1],
                    min(attn_metadata.max_seq_len, attn_metadata.topk_tokens),
                )
                if attn_metadata.num_decode_tokens == num_tokens
                else 1
            )
            if decode_num_splits > 1:
                rocm_sparse_attn_decode_bf16(
                    q=q,
                    kv=kv,
                    scale=self.scale,
                    head_dim=q.shape[-1],
                    nope_head_dim=self.kv_lora_rank,
                    rope_head_dim=q.shape[-1] - self.kv_lora_rank,
                    attn_sink=triton_sinks,
                    output=output,
                    ragged_indices=attn_metadata.paged_kv_indices,
                    ragged_indptr=attn_metadata.paged_kv_indptr,
                    num_splits=decode_num_splits,
                    kv_cache_dtype=self.kv_cache_dtype,
                )
            else:
                rocm_sparse_attn_prefill(
                    q=q,
                    kv=kv,
                    indices=None,
                    topk_length=None,
                    scale=self.scale,
                    head_dim=q.shape[-1],
                    nope_head_dim=self.kv_lora_rank,
                    rope_head_dim=q.shape[-1] - self.kv_lora_rank,
                    attn_sink=triton_sinks,
                    output=output,
                    ragged_indices=attn_metadata.paged_kv_indices,
                    ragged_indptr=attn_metadata.paged_kv_indptr,
                    kv_cache_dtype=self.kv_cache_dtype,
                )
            output = AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
            return output, None

        # AITER's nonpersistent return-LSE dispatch has discrete head kernels.
        supported_head_buckets: tuple[int, ...] | None = None
        head_dtype_name = ""
        if need_lse:
            if (
                q.dtype == torch.bfloat16
                and kv_c_and_k_pe_cache.dtype == torch.bfloat16
            ):
                supported_head_buckets = (16, 32, 64, 128)
                head_dtype_name = "BF16"
            elif (
                q.dtype == current_platform.fp8_dtype()
                and kv_c_and_k_pe_cache.dtype == current_platform.fp8_dtype()
            ):
                supported_head_buckets = (16, 128)
                head_dtype_name = "FP8"
            else:
                raise ValueError(
                    "ROCm AITER MLA attention sinks require query and KV to "
                    "both use BF16 or both use FP8, got "
                    f"query={q.dtype}, KV={kv_c_and_k_pe_cache.dtype}"
                )

        if supported_head_buckets is not None:
            from vllm.platforms.rocm import on_mi3xx

            if on_mi3xx():
                supported_heads = next(
                    (
                        heads
                        for heads in supported_head_buckets
                        if heads >= mla_num_heads
                    ),
                    None,
                )
                if supported_heads is None:
                    raise ValueError(
                        "ROCm AITER MLA attention sinks support at most 128 "
                        f"padded local {head_dtype_name} heads; increase "
                        "tensor_parallel_size"
                    )
                if supported_heads != mla_num_heads:
                    q = AiterMLAHelper.get_mla_padded_q(
                        mla_num_heads, q, supported_heads
                    )
                    mla_num_heads = supported_heads
        output = torch.empty(
            [num_tokens, mla_num_heads, self.kv_lora_rank],
            dtype=attn_metadata.attn_out_dtype,
            device=q.device,
        )

        if need_lse:
            # gfx942 has no persistent MLA code object that writes final LSE.
            # The split-KV path consumes the same ragged-index layout.
            lse = rocm_aiter_ops.mla_decode_fwd_lse(
                q,
                kv_c_and_k_pe_cache,
                output,
                self.scale,
                attn_metadata.qo_indptr,
                1,
                attn_metadata.paged_kv_indptr,
                attn_metadata.paged_kv_indices,
                attn_metadata.paged_kv_last_page_len,
                q_scale=layer._q_scale,
                kv_scale=layer._k_scale,
            )
        else:
            # Preserve the persistent work-stealing fast path for models that
            # do not use sinks.
            mla_kwargs: dict = dict(
                q_scale=layer._q_scale,
                kv_scale=layer._k_scale,
            )
            if attn_metadata.work_meta_data is not None:
                mla_kwargs.update(
                    work_meta_data=attn_metadata.work_meta_data,
                    work_indptr=attn_metadata.work_indptr,
                    work_info_set=attn_metadata.work_info_set,
                    reduce_indptr=attn_metadata.reduce_indptr,
                    reduce_final_map=attn_metadata.reduce_final_map,
                    reduce_partial_map=attn_metadata.reduce_partial_map,
                )

            rocm_aiter_ops.mla_decode_fwd(
                q,
                kv_c_and_k_pe_cache,
                output,
                self.scale,
                attn_metadata.qo_indptr,
                1,
                attn_metadata.paged_kv_indptr,
                attn_metadata.paged_kv_indices,
                attn_metadata.paged_kv_last_page_len,
                **mla_kwargs,
            )
            lse = None

        if mla_num_heads != base_mla_num_heads:
            if mla_num_heads % base_mla_num_heads == 0:
                head_stride = mla_num_heads // base_mla_num_heads
                output = output[:, ::head_stride]
                if lse is not None:
                    lse = lse[:, ::head_stride]
            else:
                output = output[:, :base_mla_num_heads]
                if lse is not None:
                    lse = lse[:, :base_mla_num_heads]

        output = AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
        if lse is not None:
            lse = AiterMLAHelper.get_mla_unpadded_o(
                self.num_heads, lse.unsqueeze(-1)
            ).squeeze(-1)

        if self.sinks is not None:
            assert lse is not None
            # Empty ragged rows have only sink mass and no value contribution.
            # AITER can return NaN output/LSE for those rows; do not multiply it
            # by a zero normalization factor and propagate the NaN.
            assert attn_metadata.paged_kv_indptr is not None
            has_keys = (
                attn_metadata.paged_kv_indptr[1:] > attn_metadata.paged_kv_indptr[:-1]
            ).unsqueeze(-1)
            lse = torch.where(has_keys, lse, float("-inf"))
            sink_lse = torch.logaddexp(lse, self.sinks)
            sink_scale = torch.exp(lse - sink_lse)
            output = torch.where(
                has_keys.unsqueeze(-1),
                output.float() * sink_scale.unsqueeze(-1),
                0.0,
            ).to(output.dtype)
            lse = sink_lse

        return output, lse

    def _forward_mla_aiter(
        self,
        layer: AttentionLayer,
        q: torch.Tensor,  # [sq, heads, d_qk], not head-padded
        kv_c_and_k_pe_cache: torch.Tensor,
        attn_metadata: ROCMAiterMLASparseMetadata,
    ) -> torch.Tensor:
        """_forward_mla on aiter's Triton sparse MLA kernel.

        Reads the same index stream, but needs no q head padding and no
        persistent metadata, and its launch depends on shapes only, so it is
        CUDA-graph capturable.
        """
        num_actual_tokens = attn_metadata.num_actual_tokens
        output = torch.empty(
            [q.shape[0], self.num_heads, self.kv_lora_rank],
            dtype=attn_metadata.attn_out_dtype,
            device=q.device,
        )
        rocm_aiter_ops.triton_sparse_mla_fwd(
            q[:num_actual_tokens],
            kv_c_and_k_pe_cache.view(-1, 1, 1, q.shape[-1]),
            output[:num_actual_tokens],
            self.scale,
            attn_metadata.paged_kv_indptr,
            attn_metadata.paged_kv_indices,
            kv_lora_rank=self.kv_lora_rank,
            qk_rope_head_dim=self.qk_rope_head_dim,
            q_scale=layer._q_scale,
            kv_scale=layer._k_scale,
            attn_sink=self.sinks,
            # triton_convert_req_index_to_global_index writes 0, never -1,
            # for an invalid top-k entry, so no slot in the stream is negative.
            has_invalid=False,
        )
        return output

    def forward_mqa(
        self,
        q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
        kv_c_and_k_pe_cache: torch.Tensor,
        attn_metadata: ROCMAiterMLASparseMetadata,
        layer: AttentionLayer,
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        # NOTE(lucas): for the sparse FlashMLA kernels the kernels want to use
        # MQA 576/512 approach for both prefill and decode

        fp8_attention = self.kv_cache_dtype.startswith("fp8")
        # fp8_ds_mla is an opaque per-token byte format (fp8 latent + per-128
        # fp32 scales + bf16 rope); only the Triton sparse MLA kernel reads it,
        # and it wants a bf16 query.
        fp8_ds_mla_cache = self.kv_cache_dtype == "fp8_ds_mla"
        if isinstance(q, tuple):
            ql_nope, q_pe = q
            if fp8_attention and not fp8_ds_mla_cache:
                q = layer._decode_concat_quant_fp8_op(  # type: ignore[attr-defined]
                    ql_nope, q_pe, layer._q_scale
                )
            else:
                q = self.q_concat_buffer[: ql_nope.shape[0]]
                if q_pe.shape[-1] == 0:
                    q.copy_(ql_nope)
                elif q.dtype == torch.float16:
                    torch.cat((ql_nope, q_pe), dim=-1, out=q)
                else:
                    ops.concat_mla_q(ql_nope, q_pe, q)

        num_actual_toks = q.shape[0]

        kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
            kv_c_and_k_pe_cache, attn_metadata.block_size
        )

        if self.dcp_world_size > 1:
            assert self.topk_indices_buffer is not None
            score_only = self.dcp_scores is not None and not attn_metadata.num_prefills
            if score_only:
                from vllm.distributed import get_dcp_group

                assert self.dcp_scores is not None

                scores = (
                    get_dcp_group()
                    .all_gather(
                        self.dcp_scores[:num_actual_toks].view(torch.int32), dim=1
                    )
                    .view(torch.float32)
                )
                token_pages = attn_metadata._dcp_token_pages
                if token_pages is None or token_pages.shape[0] != num_actual_toks:
                    token_pages = attn_metadata.block_table[
                        attn_metadata.req_id_per_token[:num_actual_toks]
                    ]
                    attn_metadata._dcp_token_pages = token_pages
                packed, indptr = merge_dcp_scores(
                    scores,
                    self.topk_indices_buffer[:num_actual_toks],
                    token_pages,
                    self.dcp_rank,
                    self.dcp_world_size,
                    attn_metadata.block_size,
                    block_stride_rows,
                )
                compact = True
            else:
                indices = triton_filter_and_convert_dcp_index(
                    attn_metadata.req_id_per_token[:num_actual_toks],
                    attn_metadata.block_table,
                    self.topk_indices_buffer[:num_actual_toks],
                    self.dcp_world_size,
                    self.dcp_rank,
                    attn_metadata.cp_kv_cache_interleave_size,
                    BLOCK_SIZE=attn_metadata.block_size,
                    BLOCK_STRIDE_ROWS=block_stride_rows,
                    NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
                    compact_valid_to_front=False,
                )
                compact = attn_metadata.num_prefills == 0
                if compact:
                    indptr, packed = _compact_dcp_indices(indices)
                else:
                    indptr = (
                        torch.arange(
                            num_actual_toks + 1, dtype=torch.int32, device=q.device
                        )
                        * attn_metadata.topk_tokens
                    )
                    packed = cast(torch.Tensor, indices).flatten()
            output = torch.empty(
                (q.shape[0], q.shape[1], self.kv_lora_rank),
                dtype=attn_metadata.attn_out_dtype,
                device=q.device,
            )
            lse = rocm_aiter_ops.triton_sparse_mla_fwd(
                q,
                kv_c_and_k_pe_cache.view(torch.uint8).unsqueeze(1),
                output,
                self.scale,
                indptr,
                packed,
                kv_lora_rank=self.kv_lora_rank,
                qk_rope_head_dim=self.qk_rope_head_dim,
                has_invalid=not compact,
                return_lse=True,
            )
            return output, lse

        # Indexer layers may rewrite their buffer, so always remap. Non-indexer
        # layers have their own KV cache group: remap once per indexer write.
        if (
            self.owns_indexer
            or attn_metadata.remapped_epoch != self.index_epoch
            or attn_metadata.remapped_buf is not self.topk_indices_buffer
        ):
            # Get topk indices
            assert self.topk_indices_buffer is not None
            topk_indices = fit_kpool_indices_to_aiter(
                self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
            )

            triton_convert_req_index_to_global_index(
                attn_metadata.req_id_per_token[:num_actual_toks],
                attn_metadata.block_table,
                topk_indices,
                attn_metadata.paged_kv_indptr,
                attn_metadata.paged_kv_indices,
                BLOCK_SIZE=attn_metadata.block_size,
                BLOCK_STRIDE_ROWS=block_stride_rows,
                NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
            )
            attn_metadata.remapped_buf = self.topk_indices_buffer
            attn_metadata.remapped_epoch = self.index_epoch

        if fp8_ds_mla_cache:
            assert self.sinks is None, (
                "the fp8_ds_mla sparse MLA path does not support attention sinks"
            )
            if q.dtype != torch.bfloat16:
                # A packed fp8 query would need a q_scale the aiter wrapper
                # does not accept; only the dequantized bf16 form is supported.
                raise NotImplementedError(
                    f"fp8_ds_mla sparse MLA requires a bf16 query, got {q.dtype}"
                )
            output = torch.empty(
                [q.shape[0], q.shape[1], self.kv_lora_rank],
                dtype=attn_metadata.attn_out_dtype,
                device=q.device,
            )
            rocm_aiter_ops.triton_sparse_mla_fwd(
                q,
                # The packed DSA record is 656 bytes wide, not the 576 of a
                # plain QK row, so aiter only recognises it as a 3-D block
                # cache. One record per block keeps the slot numbering the
                # index stream already uses.
                kv_c_and_k_pe_cache.view(torch.uint8).unsqueeze(1),
                output,
                self.scale,
                attn_metadata.paged_kv_indptr,
                attn_metadata.paged_kv_indices,
                kv_lora_rank=self.kv_lora_rank,
                qk_rope_head_dim=self.qk_rope_head_dim,
                # triton_convert_req_index_to_global_index writes 0, never -1,
                # for an invalid top-k entry, so no slot in the stream is
                # negative.
                has_invalid=False,
            )
            return output, None

        # write the latent and rope to kv cache
        if fp8_attention:
            kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(current_platform.fp8_dtype())
            if q.dtype != current_platform.fp8_dtype():
                original_q_shape = q.shape
                q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale)
                q = q.view(original_q_shape)
        if self.use_aiter_sparse_mla:
            output = self._forward_mla_aiter(
                layer, q, kv_c_and_k_pe_cache, attn_metadata
            )
            return output, None
        mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q)
        return self._forward_mla(
            layer, mla_padded_q, kv_c_and_k_pe_cache.unsqueeze(1), attn_metadata
        )

_forward_mla_aiter(layer, q, kv_c_and_k_pe_cache, attn_metadata)

_forward_mla on aiter's Triton sparse MLA kernel.

Reads the same index stream, but needs no q head padding and no persistent metadata, and its launch depends on shapes only, so it is CUDA-graph capturable.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _forward_mla_aiter(
    self,
    layer: AttentionLayer,
    q: torch.Tensor,  # [sq, heads, d_qk], not head-padded
    kv_c_and_k_pe_cache: torch.Tensor,
    attn_metadata: ROCMAiterMLASparseMetadata,
) -> torch.Tensor:
    """_forward_mla on aiter's Triton sparse MLA kernel.

    Reads the same index stream, but needs no q head padding and no
    persistent metadata, and its launch depends on shapes only, so it is
    CUDA-graph capturable.
    """
    num_actual_tokens = attn_metadata.num_actual_tokens
    output = torch.empty(
        [q.shape[0], self.num_heads, self.kv_lora_rank],
        dtype=attn_metadata.attn_out_dtype,
        device=q.device,
    )
    rocm_aiter_ops.triton_sparse_mla_fwd(
        q[:num_actual_tokens],
        kv_c_and_k_pe_cache.view(-1, 1, 1, q.shape[-1]),
        output[:num_actual_tokens],
        self.scale,
        attn_metadata.paged_kv_indptr,
        attn_metadata.paged_kv_indices,
        kv_lora_rank=self.kv_lora_rank,
        qk_rope_head_dim=self.qk_rope_head_dim,
        q_scale=layer._q_scale,
        kv_scale=layer._k_scale,
        attn_sink=self.sinks,
        # triton_convert_req_index_to_global_index writes 0, never -1,
        # for an invalid top-k entry, so no slot in the stream is negative.
        has_invalid=False,
    )
    return output

ROCMAiterMLASparseMetadataBuilder dataclass

Bases: SparseMLACommonMetadataBuilder[ROCMAiterMLASparseMetadata]

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
@dataclass
class ROCMAiterMLASparseMetadataBuilder(
    SparseMLACommonMetadataBuilder[ROCMAiterMLASparseMetadata]
):
    _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
    metadata_cls = ROCMAiterMLASparseMetadata

    def __init__(
        self,
        kv_cache_spec: AttentionSpec,
        layer_names: list[str],
        vllm_config: VllmConfig,
        device: torch.device,
    ):
        super().__init__(kv_cache_spec, layer_names, vllm_config, device)
        self.model_dtype = vllm_config.model_config.dtype
        self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
        parallel_config = vllm_config.parallel_config
        max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens

        self._init_reorder_batch_threshold(
            1, supports_spec_as_decode=True, supports_dcp_with_varlen=True
        )

        self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
        attention_context = vllm_config.compilation_config.static_forward_context
        # Sink decode must use AITER's nonpersistent path. In particular,
        # gfx942 has no persistent+LSE kernel, and its metadata heuristic
        # terminates for HY-V4's TP1 H64 shape. The Triton sparse MLA kernel
        # never reads the persistent metadata either.
        # fp8_ds_mla never reads it either: its forward_mqa branch always
        # routes to aiter's Triton sparse_mla_fwd.
        self._use_persistent_metadata = all(
            getattr(attention_context[name].impl, "sinks", None) is None
            and not getattr(attention_context[name].impl, "use_aiter_sparse_mla", False)
            and getattr(attention_context[name].impl, "kv_cache_dtype", None)
            != "fp8_ds_mla"
            for name in layer_names
        )
        # Bounds the KV-split heuristic (see `_sparse_decode_max_split`).
        self._num_compute_units = current_platform.num_compute_units()
        self.max_model_len_tensor = torch.tensor(
            [self.model_config.max_model_len], device=device, dtype=torch.int32
        )
        # this is ignored by `flash_mla_with_kvcache` if indices not None
        self.dummy_block_table = torch.empty(
            (1, 1), dtype=torch.int32, device=self.device
        )

        self.req_id_per_token_buffer = torch.zeros(
            (vllm_config.scheduler_config.max_num_batched_tokens,),
            dtype=torch.int32,
            device=device,
        )
        self.qo_indptr = torch.arange(
            0, max_num_batched_tokens + 1, dtype=torch.int32, device=device
        )
        self.paged_kv_last_page_len = torch.ones(
            max_num_batched_tokens, dtype=torch.int32, device=device
        )

        # These two needs to be calculated in runtime,
        # but we still needs to prepare the buffer
        self.paged_kv_indices = torch.zeros(
            [max_num_batched_tokens * self.topk_tokens],
            dtype=torch.int32,
            device=device,
        )
        self.paged_kv_indptr = torch.zeros(
            [max_num_batched_tokens + 1], dtype=torch.int32, device=device
        )

        # ----- Persistent MLA metadata buffers -----
        # The aiter sparse decode kernel supports a "persistent" path that
        # uses precomputed work-splitting metadata for better load balancing
        # across CUs. Mirrors the approach used in rocm_aiter_mla.py.
        #
        # In the sparse case each query token is its own "batch" entry in the
        # qo_indptr (qo_indptr = [0, 1, 2, ..., num_tokens]) and max_qo_len=1.
        # We pad get_mla_metadata_info_v1's batch_size to max_num_batched_tokens
        # so the buffers are large enough for any decode shape we might see.
        from aiter import dtypes, get_mla_metadata_info_v1

        # Keep metadata sizing consistent with the padded tensor shape passed
        # to the sparse decode kernel.
        self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
            self.num_heads
        )

        q_dtype = self.model_dtype
        kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
        if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
            kv_cache_dtype_str = "fp8"
            q_dtype = dtypes.fp8
        else:
            kv_cache_dtype_str = "bf16"
        kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16)
        self._mla_q_dtype = q_dtype
        self._mla_kv_dtype = kv_dtype

        (
            (work_meta_data_size, work_meta_data_type),
            (work_indptr_size, work_indptr_type),
            (work_info_set_size, work_info_set_type),
            (reduce_indptr_size, reduce_indptr_type),
            (reduce_final_map_size, reduce_final_map_type),
            (reduce_partial_map_size, reduce_partial_map_type),
        ) = get_mla_metadata_info_v1(
            max_num_batched_tokens,
            1,
            self._num_attention_heads,
            q_dtype,
            kv_dtype,
            is_sparse=True,
            fast_mode=True,
        )
        self._mla_work_meta_data = torch.empty(
            work_meta_data_size, dtype=work_meta_data_type, device=device
        )
        self._mla_work_indptr = torch.empty(
            work_indptr_size, dtype=work_indptr_type, device=device
        )
        self._mla_work_info_set = torch.empty(
            work_info_set_size, dtype=work_info_set_type, device=device
        )
        self._mla_reduce_indptr = torch.empty(
            reduce_indptr_size, dtype=reduce_indptr_type, device=device
        )
        self._mla_reduce_final_map = torch.empty(
            reduce_final_map_size, dtype=reduce_final_map_type, device=device
        )
        self._mla_reduce_partial_map = torch.empty(
            reduce_partial_map_size,
            dtype=reduce_partial_map_type,
            device=device,
        )

        self._prev_req_extent: int = 0
        self._prev_indices_extent: int = 0
        self._prev_metadata_key: tuple | None = None

    def _sparse_decode_max_split(self, max_seq_len: int) -> int:
        """Cap `max_split_per_batch` for the aiter sparse-MLA decode reduce.

        The reduce only covers the selected tokens per row (`<= topk_tokens`),
        so aiter's default (`-1` => split across every CU) over-fragments it.
        Mirror `triton_mla.py`: aim for a minimum work per split, round to a
        power of two, and cap by the CU count. Numerics are unchanged.
        """
        effective_len = min(max_seq_len, self.topk_tokens)
        min_work_per_split = 128
        ideal_splits = triton.next_power_of_2(
            max(1, effective_len // min_work_per_split)
        )
        return min(ideal_splits, self._num_compute_units)

    def build(
        self,
        common_prefix_len: int,
        common_attn_metadata: CommonAttentionMetadata,
        fast_build: bool = False,
    ) -> ROCMAiterMLASparseMetadata:
        metadata = super().build(
            common_prefix_len, common_attn_metadata, fast_build=fast_build
        )
        use_dense_prefill = (
            metadata.prefill is not None and metadata.prefill.use_dense_mha
        )
        num_reqs = metadata.num_decodes if use_dense_prefill else metadata.num_reqs
        num_tokens = int(common_attn_metadata.query_start_loc_cpu[num_reqs].item())
        num_decodes = metadata.num_decodes
        num_prefills = metadata.num_prefills
        num_decode_tokens = metadata.num_decode_tokens
        starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
        seg_lengths = np.diff(starts[: num_reqs + 1])
        req_id_per_token = np.repeat(
            np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
        )
        # Only re-zero the shrink-tail. paged_kv_indptr is fully rewritten
        # by the cumsum below. paged_kv_indices entries past new_indices_extent
        # are never read (the attention kernel only touches the ranges
        # defined by paged_kv_indptr).
        new_req_extent = int(req_id_per_token.shape[0])
        new_indices_extent = num_tokens * self.topk_tokens
        if self._prev_req_extent > new_req_extent:
            self.req_id_per_token_buffer[new_req_extent : self._prev_req_extent].fill_(
                0
            )
        # DCP attention builds its own CSR indices: skip the paged ones.
        dcp = self.vllm_config.parallel_config.decode_context_parallel_size > 1
        if not dcp and self._prev_indices_extent > new_indices_extent:
            self.paged_kv_indices[new_indices_extent : self._prev_indices_extent].fill_(
                0
            )
        self._prev_req_extent = new_req_extent
        self._prev_indices_extent = new_indices_extent
        self.req_id_per_token_buffer[:new_req_extent].copy_(
            np_to_pinned_tensor(req_id_per_token), non_blocking=True
        )
        if not dcp:
            query_start_loc = common_attn_metadata.query_start_loc[: num_reqs + 1]
            seq_lens = common_attn_metadata.seq_lens[:num_reqs]
            sparse_seqlen = generate_sparse_seqlen_triton(
                seq_lens,
                query_start_loc,
                self.topk_tokens,
                num_tokens,
                common_attn_metadata.max_query_len,
            )

            torch.cumsum(
                sparse_seqlen, dim=0, out=self.paged_kv_indptr[1 : num_tokens + 1]
            )
            # fill_() with a value that aliases the destination clones it first.
            indptr_tail = self.paged_kv_indptr[num_tokens + 1 :]
            indptr_tail.copy_(self.paged_kv_indptr[num_tokens].expand_as(indptr_tail))

        req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
        qo_indptr = self.qo_indptr[: num_tokens + 1]
        paged_kv_last_page_len = self.paged_kv_last_page_len[:num_tokens]
        paged_kv_indptr = self.paged_kv_indptr[: num_tokens + 1]
        paged_kv_indices = self.paged_kv_indices[: num_tokens * self.topk_tokens]

        # ----- Compute persistent MLA metadata -----
        # The AITER sparse decode kernel uses qseqlen=1 (each query token is
        # treated as its own batch entry). Build its persistent work metadata
        # only when AITER is selected and no layer needs the nonpersistent LSE
        # path for attention sinks. The output is a deterministic function of
        # the per-request query and context lengths (both clamped to
        # topk_tokens, past which per-token KV length saturates) and num_heads;
        # fingerprint those CPU-side and skip the launch when nothing changed.
        head_size = self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim
        use_triton_sparse = _use_rocm_sparse_triton(
            kv_cache_dtype=self.kv_cache_dtype,
            head_size=head_size,
            kv_lora_rank=self.mla_dims.kv_lora_rank,
            num_prefills=num_prefills,
            num_decodes=num_decodes,
            num_decode_tokens=num_decode_tokens,
            max_query_len=common_attn_metadata.max_query_len,
        )
        work_meta_data = None
        work_indptr = None
        work_info_set = None
        reduce_indptr = None
        reduce_final_map = None
        reduce_partial_map = None
        if (
            self._use_persistent_metadata
            and not dcp
            and not use_triton_sparse
            and not _use_triton_long_bf16_prefill(self._mla_kv_dtype, metadata)
            and num_tokens > 0
            and _aiter_uses_persistent_mla_decode(
                num_tokens,
                self._num_attention_heads,
                self._mla_q_dtype,
                self._mla_kv_dtype,
            )
        ):
            with gpu_sync_allowed():
                seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy()
            clamped_seq_lens = np.minimum(
                seq_lens_cpu,
                self.topk_tokens,
            )
            clamped_context_lens = np.minimum(
                seq_lens_cpu - seg_lengths,
                self.topk_tokens,
            )
            metadata_key = (
                num_tokens,
                int(common_attn_metadata.max_query_len),
                self._num_attention_heads,
                clamped_seq_lens.tobytes(),
                clamped_context_lens.tobytes(),
                seg_lengths.tobytes(),
            )
            if metadata_key != self._prev_metadata_key:
                from aiter import get_mla_metadata_v1

                max_split_per_batch = self._sparse_decode_max_split(
                    int(common_attn_metadata.max_seq_len)
                )
                get_mla_metadata_v1(
                    qo_indptr,
                    paged_kv_indptr,
                    paged_kv_last_page_len,
                    self._num_attention_heads,
                    1,
                    True,
                    self._mla_work_meta_data,
                    self._mla_work_info_set,
                    self._mla_work_indptr,
                    self._mla_reduce_indptr,
                    self._mla_reduce_final_map,
                    self._mla_reduce_partial_map,
                    page_size=1,
                    kv_granularity=16,
                    max_seqlen_qo=1,
                    uni_seqlen_qo=1,
                    fast_mode=True,
                    max_split_per_batch=max_split_per_batch,
                    dtype_q=self._mla_q_dtype,
                    dtype_kv=self._mla_kv_dtype,
                )
                self._prev_metadata_key = metadata_key
            work_meta_data = self._mla_work_meta_data
            work_indptr = self._mla_work_indptr
            work_info_set = self._mla_work_info_set
            reduce_indptr = self._mla_reduce_indptr
            reduce_final_map = self._mla_reduce_final_map
            reduce_partial_map = self._mla_reduce_partial_map

        metadata.attn_out_dtype = self.model_dtype
        metadata.req_id_per_token = req_id_per_token
        metadata.qo_indptr = qo_indptr
        metadata.paged_kv_last_page_len = paged_kv_last_page_len
        metadata.paged_kv_indices = paged_kv_indices
        metadata.paged_kv_indptr = paged_kv_indptr
        metadata.work_meta_data = work_meta_data
        metadata.work_indptr = work_indptr
        metadata.work_info_set = work_info_set
        metadata.reduce_indptr = reduce_indptr
        metadata.reduce_final_map = reduce_final_map
        metadata.reduce_partial_map = reduce_partial_map
        return metadata

_sparse_decode_max_split(max_seq_len)

Cap max_split_per_batch for the aiter sparse-MLA decode reduce.

The reduce only covers the selected tokens per row (<= topk_tokens), so aiter's default (-1 => split across every CU) over-fragments it. Mirror triton_mla.py: aim for a minimum work per split, round to a power of two, and cap by the CU count. Numerics are unchanged.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _sparse_decode_max_split(self, max_seq_len: int) -> int:
    """Cap `max_split_per_batch` for the aiter sparse-MLA decode reduce.

    The reduce only covers the selected tokens per row (`<= topk_tokens`),
    so aiter's default (`-1` => split across every CU) over-fragments it.
    Mirror `triton_mla.py`: aim for a minimum work per split, round to a
    power of two, and cap by the CU count. Numerics are unchanged.
    """
    effective_len = min(max_seq_len, self.topk_tokens)
    min_work_per_split = 128
    ideal_splits = triton.next_power_of_2(
        max(1, effective_len // min_work_per_split)
    )
    return min(ideal_splits, self._num_compute_units)

_aiter_uses_persistent_mla_decode(num_tokens, num_heads, q_dtype, kv_dtype)

AITER ignores persistent metadata for some gfx950 BF16 16-head batches.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _aiter_uses_persistent_mla_decode(
    num_tokens: int, num_heads: int, q_dtype: torch.dtype, kv_dtype: torch.dtype
) -> bool:
    """AITER ignores persistent metadata for some gfx950 BF16 16-head batches."""
    try:
        from aiter.mla import _use_persistent_mla_decode
    except ImportError:
        return True
    return _use_persistent_mla_decode(num_tokens, num_heads, 1, q_dtype, kv_dtype)

_compact_dcp_indices(indices, counts=None)

Preserve each row's valid slot order while removing invalid KV entries.

Parameters:

  • indices

    (Tensor) –

    Per-row KV rows, where -1 marks an invalid entry.

  • counts

    (Tensor | None, default: None ) –

    Optional per-row count of valid entries, which must then be a prefix of the row. Pass it to save the separate prefix sum.

Returns:

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _compact_dcp_indices(
    indices: torch.Tensor, counts: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
    """Preserve each row's valid slot order while removing invalid KV entries.

    Args:
        indices: Per-row KV rows, where `-1` marks an invalid entry.
        counts: Optional per-row count of valid entries, which must then be a
            prefix of the row. Pass it to save the separate prefix sum.

    Returns:
        The CSR row pointer and the packed rows.

    """
    rows, topk = indices.shape
    if counts is None:
        indptr = torch.zeros(rows + 1, dtype=torch.int32, device=indices.device)
        valid = (indices >= 0).sum(dim=1, dtype=torch.int32)
        torch.cumsum(valid, dim=0, out=indptr[1:])
    else:
        # _pack_valid_rows fills indptr, but it does not run without rows.
        alloc = torch.empty if rows else torch.zeros
        indptr = alloc(rows + 1, dtype=torch.int32, device=indices.device)
    packed = torch.empty(indices.numel(), dtype=indices.dtype, device=indices.device)
    if rows:
        _pack_valid_rows[(rows,)](
            indices,
            indices.stride(0),
            indptr,
            packed,
            counts,
            rows,
            topk,
            triton.next_power_of_2(topk),
            0 if counts is None else max(16, triton.next_power_of_2(rows)),
        )
    return indptr, packed

_index_epoch(buf, owns_indexer)

Number of indexer writes up to and including this layer.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _index_epoch(buf: object, owns_indexer: bool) -> int:
    """Number of indexer writes up to and including this layer."""
    # id(): tensors are unhashable, and the buffer outlives every reader.
    epoch = _INDEX_EPOCHS.get(id(buf), 0) + (1 if owns_indexer else 0)
    _INDEX_EPOCHS[id(buf)] = epoch
    return epoch

_use_rocm_sparse_triton(*, kv_cache_dtype, head_size, kv_lora_rank, num_prefills, num_decodes, num_decode_tokens, max_query_len)

Select the rope-free BF16 path not supported by AITER sparse MLA.

The ragged Triton kernel indexes metadata per query token, so multi-token speculative verification rows have the same capability requirements as plain decode rows.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _use_rocm_sparse_triton(
    *,
    kv_cache_dtype: str,
    head_size: int,
    kv_lora_rank: int,
    num_prefills: int,
    num_decodes: int,
    num_decode_tokens: int,
    max_query_len: int,
) -> bool:
    """Select the rope-free BF16 path not supported by AITER sparse MLA.

    The ragged Triton kernel indexes metadata per query token, so multi-token
    speculative verification rows have the same capability requirements as
    plain decode rows.
    """
    return (
        not kv_cache_dtype.startswith("fp8")
        and head_size == kv_lora_rank
        and (num_prefills > 0 or num_decodes > 0)
    )

_use_triton_long_bf16_prefill(kv_dtype, metadata)

Only gfx942 routes long pure BF16 prefills to Triton.

Other archs must keep persistent AITER metadata for these batches.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def _use_triton_long_bf16_prefill(
    kv_dtype: torch.dtype,
    metadata: "ROCMAiterMLASparseMetadata",
) -> bool:
    """Only gfx942 routes long pure BF16 prefills to Triton.

    Other archs must keep persistent AITER metadata for these batches.
    """
    from vllm.platforms.rocm import on_gfx942

    return (
        on_gfx942()
        and kv_dtype == torch.bfloat16
        and metadata.num_decodes == 0
        and metadata.num_prefills > 0
        and metadata.prefill_max_seq_len > metadata.topk_tokens
    )

fit_kpool_indices_to_aiter(token_indices, topk_tokens)

Keep the live kpool tail while fitting AITER's fixed top-k width.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def fit_kpool_indices_to_aiter(
    token_indices: torch.Tensor, topk_tokens: int
) -> torch.Tensor:
    """Keep the live kpool tail while fitting AITER's fixed top-k width."""
    if token_indices.shape[1] < topk_tokens:
        raise ValueError("token_indices width must be at least topk_tokens")
    if token_indices.shape[1] == topk_tokens:
        return token_indices

    num_tokens, width = token_indices.shape
    tail_width = width - topk_tokens
    output = torch.empty(
        (num_tokens, topk_tokens),
        dtype=token_indices.dtype,
        device=token_indices.device,
    )
    if num_tokens == 0:
        return output

    _fit_kpool_indices_kernel[(num_tokens,)](
        token_indices,
        output,
        token_indices.stride(0),
        token_indices.stride(1),
        output.stride(0),
        output.stride(1),
        NUM_TOPK_TOKENS=topk_tokens,
        TAIL_WIDTH=tail_width,
        BLOCK_T=triton.next_power_of_2(topk_tokens),
        BLOCK_TAIL=triton.next_power_of_2(tail_width),
    )
    return output

merge_dcp_scores(scores, local_indices, token_block_table, rank, world_size, page_size, block_stride_rows)

Merge rank-major score candidates into this rank's physical CSR slots.

Same selection as aiter flydsl_dcp_topk_merge: the global top-k by score, ties at the k-th score taken in rank-major position order.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def merge_dcp_scores(
    scores: torch.Tensor,
    local_indices: torch.Tensor,
    token_block_table: torch.Tensor,
    rank: int,
    world_size: int,
    page_size: int,
    block_stride_rows: int,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Merge rank-major score candidates into this rank's physical CSR slots.

    Same selection as aiter `flydsl_dcp_topk_merge`: the global top-k by
    score, ties at the k-th score taken in rank-major position order.
    """
    rows, k = local_indices.shape
    width = scores.shape[1]
    assert width == world_size * k and scores.stride(1) == 1
    ends = _score_widths.get((scores.device, width))
    if ends is None or ends.numel() < rows:
        ends = torch.full(
            (max(rows, 1024),), width, dtype=torch.int32, device=scores.device
        )
        _score_widths[(scores.device, width)] = ends
    # Only the k-th score is used: any tie order is fine here.
    top = torch.empty(local_indices.shape, dtype=torch.float32, device=scores.device)
    rocm_aiter_ops.indexer_top_k_decode(
        scores,
        1,
        ends,
        torch.empty_like(local_indices),
        k,
        values=top,
    )
    staging = torch.empty_like(local_indices)
    counts = torch.empty(rows, dtype=torch.int32, device=scores.device)
    _select_owned_slots[(rows,)](
        scores,
        top,
        local_indices,
        token_block_table,
        staging,
        counts,
        rank,
        scores.stride(0),
        top.stride(0),
        local_indices.stride(0),
        token_block_table.stride(0),
        k,
        page_size,
        block_stride_rows,
        num_warps=8,
    )
    indptr, slots = _compact_dcp_indices(staging, counts)
    return slots, indptr

triton_convert_req_index_to_global_index(req_id, block_table, token_indices, cu_seqlens, paged_kv_indices, BLOCK_STRIDE_ROWS, BLOCK_SIZE=64, NUM_TOPK_TOKENS=2048, BLOCK_N=128)

out[token_id, indice_id] = block_table[req_id[token_id], token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_STRIDE_ROWS + token_indices[token_id, indice_id] % BLOCK_SIZE

Only when token_indices[token_id, indice_id] == -1 do we output -1. For safety, we also output -1 if the derived block_id would be out-of-bounds.

Source code in vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
def triton_convert_req_index_to_global_index(
    req_id: torch.Tensor,  # int32 [num_tokens]
    block_table: torch.Tensor,  # int32 [num_requests, max_num_blocks_per_req]
    token_indices: torch.Tensor,  # int32 [num_tokens, NUM_TOPK_TOKENS]
    cu_seqlens: torch.Tensor,  # int32 [num_tokens + 1]
    paged_kv_indices: torch.Tensor,  # int32 [num_tokens * topk] out_buffer
    BLOCK_STRIDE_ROWS: int,
    BLOCK_SIZE: int = 64,
    NUM_TOPK_TOKENS: int = 2048,
    BLOCK_N: int = 128,  # tile width along columns
):
    """out[token_id, indice_id] =
        block_table[req_id[token_id],
            token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_STRIDE_ROWS
        + token_indices[token_id, indice_id] % BLOCK_SIZE

    Only when token_indices[token_id, indice_id] == -1 do we output -1.
    For safety, we also output -1 if the derived block_id would be
        out-of-bounds.
    """
    assert req_id.dtype == torch.int32
    assert block_table.dtype == torch.int32
    assert token_indices.dtype == torch.int32
    assert req_id.shape[0] == token_indices.shape[0], (
        f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) "
        "must cover the same tokens"
    )
    assert token_indices.shape[1] == NUM_TOPK_TOKENS
    assert NUM_TOPK_TOKENS % BLOCK_N == 0, (
        f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible byBLOCK_N ({BLOCK_N})"
    )
    # print("req_id: ", req_id, flush=True)
    num_tokens = req_id.shape[0]
    _, max_num_blocks_per_req = block_table.shape
    tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N

    # Ensure contiguous tensors on the same device
    req_id_c = req_id.contiguous()
    block_table_c = block_table.contiguous()
    token_indices_c = token_indices.contiguous()

    # Strides in elements
    bt_stride0, bt_stride1 = block_table_c.stride()
    ti_stride0, ti_stride1 = token_indices_c.stride()

    # Exact 2D grid: tokens × column tiles
    grid = (num_tokens, tiles_per_row)

    _convert_req_index_to_global_index_kernel[grid](
        req_id_c,
        block_table_c,
        token_indices_c,
        cu_seqlens,
        paged_kv_indices,
        # shapes / constexprs
        max_num_blocks_per_req,
        BLOCK_SIZE,
        BLOCK_STRIDE_ROWS,
        BLOCK_N,
        # strides
        bt_stride0,
        bt_stride1,
        ti_stride0,
        ti_stride1,
    )
    return