Skip to content

vllm.model_executor.layers.indexer_topk

Top-k kernels for the DSA sparse attention indexer.

Classes:

Functions:

SparseIndexerTopk

Bases: Module

The sparse indexer's decode top-k stage.

Selects among the available top-k kernels (see kernel_config.sparse_indexer_topk_backend) and runs the chosen one.

Methods:

  • forward –

    Run the resolved decode top-k implementation, writing into

  • forward_prefill –

    Prefill top-k: row r selects over

  • resolve_backend –

    Resolve the decode top-k implementation from the configured

  • resolve_prefill_backend –

    flashinfer_gvr2 for chunks of at least GVR2_PREFILL_MIN_ROWS

  • takes_bf16_logits –

    Whether this decode call can consume BF16 without a full FP32 copy.

Source code in vllm/model_executor/layers/indexer_topk.py
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
class SparseIndexerTopk(torch.nn.Module):
    """The sparse indexer's decode top-k stage.

    Selects among the available top-k kernels (see
    kernel_config.sparse_indexer_topk_backend) and runs the chosen one.
    """

    def __init__(self, backend: str | None = None) -> None:
        super().__init__()
        vllm_config = get_current_vllm_config_or_none()
        if backend is None:
            if vllm_config is None:
                vllm_config = get_current_vllm_config()
            backend = vllm_config.kernel_config.sparse_indexer_topk_backend
        self._backend = backend
        # Decode rows per step (requests x tokens per request), used to warm
        # up gvr_2 for every row count before the first CUDA-graph capture.
        self._gvr2_max_rows = 256
        if vllm_config is not None:
            spec = vllm_config.speculative_config
            self._gvr2_max_rows = vllm_config.scheduler_config.max_num_seqs * (
                1 + (spec.num_speculative_tokens if spec is not None else 0)
            )
        self._gvr2_warmed: set[tuple[int, int, int]] = set()
        self._is_cuda = current_platform.is_cuda()
        self._has_deep_select = self._is_cuda and (
            current_platform.is_device_capability_family(100)
        )
        self._has_flashinfer_topk = has_flashinfer()
        self._has_flashinfer_gvr2 = (
            self._is_cuda
            and current_platform.is_device_capability_family(100)
            and has_flashinfer_topk_varlen_gvr2()
        )
        self._cooperative_capable = self._is_cuda and (
            current_platform.has_device_capability(90)
            and not current_platform.is_device_capability_family(110)
            and not current_platform.is_device_capability_family(120)
        )
        self._is_rocm = current_platform.is_rocm()
        self._aiter_enabled = False
        self._aiter_capable = False
        if self._is_rocm:
            from vllm._aiter_ops import rocm_aiter_ops

            self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
            self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())

    def resolve_backend(
        self, logits: torch.Tensor, topk_tokens: int, num_rows: int
    ) -> str:
        """Resolve the decode top-k implementation from the configured
        backend ("auto" = the pre-existing chain, or a validated explicit
        value)."""
        if self._backend == "auto":
            return self._resolve_auto(logits, topk_tokens, num_rows)

        failures: list[str] = []
        if self._backend == "cooperative":
            failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
        elif self._backend == "persistent":
            if not (self._is_cuda or self._is_rocm):
                failures.append("requires a CUDA or ROCm platform")
            if topk_tokens not in (512, 1024, 2048):
                failures.append(
                    f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
                )
        elif self._backend == "deep_select":
            if not self._is_cuda:
                failures.append("requires a CUDA platform")
            elif not self._has_deep_select:
                failures.append("requires SM100a/SM103a (10.x device family)")
            elif not _HAS_DEEP_SELECT_C:
                failures.append("vllm._deepselect_C could not be imported")
            elif not is_deep_select_supported(logits, topk_tokens):
                failures.append(
                    f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
                    f" stride={logits.stride()}, topk={topk_tokens}"
                )
        elif self._backend == "aiter":
            if not self._is_rocm:
                failures.append("requires a ROCm platform")
            elif not self._aiter_enabled:
                failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
            elif not self._aiter_capable:
                failures.append(
                    "AITER has no tuned indexer top-k kernels for this GPU arch"
                )
            if topk_tokens not in (512, 1024, 2048, 4096):
                failures.append(
                    f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
                )
        elif self._backend == "flashinfer":
            if not self._is_cuda:
                failures.append("requires a CUDA platform")
            if not self._has_flashinfer_topk:
                failures.append(
                    "flashinfer.topk.top_k_ragged_transform is not importable"
                )
            if logits.dtype != torch.float32 or logits.stride(1) != 1:
                failures.append(
                    f"requires fp32 logits with stride(1) == 1, got "
                    f"dtype={logits.dtype}, stride={logits.stride()}"
                )
        elif self._backend == "flashinfer_gvr2":
            if not self._is_cuda:
                failures.append("requires a CUDA platform")
            elif not current_platform.is_device_capability_family(100):
                failures.append("requires the SM100 device family")
            elif not self._has_flashinfer_gvr2:
                failures.append("flashinfer.top_k_varlen has no gvr_2 backend")
            failures += self._gvr2_constraints(logits, topk_tokens)
        if failures:
            raise RuntimeError(
                f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
                + "; ".join(failures)
            )
        return self._backend

    def takes_bf16_logits(
        self, logits: torch.Tensor, topk_tokens: int, num_rows: int
    ) -> bool:
        """Whether this decode call can consume BF16 without a full FP32 copy."""
        if logits.dtype != torch.bfloat16 or logits.stride(1) != 1:
            return False
        if self._backend == "deep_select":
            return (
                self._has_deep_select
                and _HAS_DEEP_SELECT_C
                and is_deep_select_supported(logits, topk_tokens)
            )
        if self._backend == "cooperative":
            return not self._cooperative_constraints(logits, topk_tokens, num_rows)
        if (
            self._backend == "auto"
            and self._resolve_auto(logits, topk_tokens, num_rows) == "cooperative"
        ):
            return True
        return (
            self._backend in ("auto", "persistent")
            and self._is_cuda
            and current_platform.is_device_capability_family(100)
            and topk_tokens in (512, 1024, 2048)
            and self.resolve_backend(logits, topk_tokens, num_rows) == "persistent"
        )

    def _resolve_auto(
        self, logits: torch.Tensor, topk_tokens: int, num_rows: int
    ) -> str:
        """The priority chain: cooperative -> persistent -> per_row.
        aiter/deep_select/flashinfer/flashinfer_gvr2/torch are opt-in only.

        cooperative_topk is preferred whenever it is applicable, i.e. within
        its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
        persistent_topk.

        aiter not auto-resolved as it does not outperform the per-row on every shape,
        hence we delegate to the caller to decide when to use it.
        """
        if not self._cooperative_constraints(logits, topk_tokens, num_rows):
            return "cooperative"
        if self._is_cuda and topk_tokens in (512, 1024, 2048):
            return "persistent"
        return "per_row"

    def _gvr2_warmup(self, logits: torch.Tensor, topk_tokens: int) -> None:
        """Compile gvr_2's kernel variants for every decode row count once.

        The kernel family and template depend on the row count, and the
        indexer runs eagerly (outside CUDA graphs) in some models, so without
        this every new batch size would JIT-compile (~1 s) in the middle of
        serving. Must run eagerly; the launchers are cached per row count.
        """
        key = (logits.shape[1], logits.stride(0), topk_tokens)
        if key in self._gvr2_warmed:
            return
        from flashinfer.topk_varlen.kernels.gvr2_topk_host import warmup_varlen

        # warmup_varlen allocates rows x stride fp32 once; keep it under 256MB.
        rows_cap = max(1, (256 << 20) // (logits.stride(0) * 4))
        max_rows = max(logits.shape[0], min(self._gvr2_max_rows, rows_cap))
        start = time.perf_counter()
        warmup_varlen(
            top_k=topk_tokens,
            max_seq_len=logits.shape[1],
            compress_ratio=1,
            next_n=1,
            num_rows_list=range(1, max_rows + 1),
            row_stride=logits.stride(0),
        )
        self._gvr2_warmed.add(key)
        logger.info_once(
            "Warmed up gvr_2 top-k for up to %d rows x %d (topk %d) in %.1fs",
            max_rows,
            logits.shape[1],
            topk_tokens,
            time.perf_counter() - start,
        )

    @staticmethod
    def _gvr2_constraints(logits: torch.Tensor, topk_tokens: int) -> list[str]:
        """Unmet input constraints of gvr_2 (empty when applicable)."""
        failures = []
        if topk_tokens not in (512, 1024, 2048):
            failures.append(
                f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
            )
        if logits.dtype != torch.float32 or logits.stride(1) != 1:
            failures.append(
                f"requires fp32 logits with stride(1) == 1, got "
                f"dtype={logits.dtype}, stride={logits.stride()}"
            )
        # float4 row loads: the row pitch must be a multiple of 4 elements and
        # the base 16-byte aligned; rows may not overlap (an expand view).
        num_rows, width = logits.shape
        row_pitch = logits.stride(0) if num_rows > 1 else width
        if row_pitch % 4 != 0 or (num_rows > 1 and row_pitch < width):
            failures.append(
                f"logits row stride must be a multiple of 4 and at least the "
                f"row width, got stride(0)={row_pitch}, width={width}"
            )
        elif num_rows > 1 and logits.untyped_storage().nbytes() < (
            (logits.storage_offset() + num_rows * row_pitch) * logits.element_size()
        ):
            failures.append("logits storage must cover every row's full stride")
        if logits.data_ptr() % 16 != 0:
            failures.append("logits base pointer must be 16-byte aligned")
        return failures

    def resolve_prefill_backend(
        self,
        logits: torch.Tensor,
        topk_tokens: int,
        row_starts: torch.Tensor | None,
    ) -> str:
        """flashinfer_gvr2 for chunks of at least GVR2_PREFILL_MIN_ROWS
        row-relative rows (row_starts is None) when it was requested
        explicitly, otherwise per_row (top_k_per_row_prefill).

        gvr_2 reads every row from column 0. Unmet gvr_2 constraints fall
        back instead of raising, since the same backend setting also drives
        decode.
        """
        if (
            self._backend == "flashinfer_gvr2"
            and row_starts is None
            and logits.shape[0] >= GVR2_PREFILL_MIN_ROWS
            and self._has_flashinfer_gvr2
            and not self._gvr2_constraints(logits, topk_tokens)
        ):
            return "flashinfer_gvr2"
        return "per_row"

    def forward_prefill(
        self,
        logits: torch.Tensor,
        row_starts: torch.Tensor | None,
        row_ends: torch.Tensor,
        topk_indices: torch.Tensor,
        topk_tokens: int,
    ) -> None:
        """Prefill top-k: row `r` selects over
        `logits[r, row_starts[r]:row_ends[r]]` (row_starts None means 0) and
        writes indices relative to the row start (int32, -1 fill for rows
        shorter than topk_tokens)."""
        num_rows = logits.shape[0]
        backend = self.resolve_prefill_backend(logits, topk_tokens, row_starts)
        if backend == "flashinfer_gvr2":
            logger.info_once("Sparse indexer prefill top-k backend: flashinfer_gvr2")
            from flashinfer.topk_varlen.kernels.gvr2_topk_host import run_varlen

            direct = topk_indices.is_contiguous() and topk_indices.data_ptr() % 16 == 0
            out = topk_indices if direct else torch.empty_like(topk_indices)
            # max_seq_len=row width avoids the host read of row_ends.max().
            run_varlen(
                logits,
                None,
                row_ends.contiguous(),
                out,
                next_n=1,
                compress_ratio=1,
                max_seq_len=logits.shape[1],
                top_k=topk_tokens,
            )
            if not direct:
                topk_indices.copy_(out)
            return
        ops.top_k_per_row_prefill(
            logits,
            torch.zeros_like(row_ends) if row_starts is None else row_starts,
            row_ends,
            topk_indices,
            num_rows,
            logits.stride(0),
            logits.stride(1),
            topk_tokens,
        )

    def _cooperative_constraints(
        self, logits: torch.Tensor, topk_tokens: int, num_rows: int
    ) -> list[str]:
        """Unmet constraints of cooperative_topk (empty when applicable)."""
        failures = []
        if not self._is_cuda:
            failures.append("requires a CUDA platform")
        if topk_tokens not in (512, 1024, 2048):
            failures.append(
                f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
            )
        if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
            failures.append(
                f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
            )
        if logits.dtype not in (torch.float32, torch.bfloat16):
            failures.append(f"logits must be fp32 or bf16, got {logits.dtype}")
        # Rows are TMA-loaded in 16-byte units.
        align = 16 // logits.element_size()
        if logits.stride(0) % align != 0:
            failures.append(
                f"logits.stride(0) must be divisible by {align}, got {logits.stride(0)}"
            )
        if self._is_cuda and not self._cooperative_capable:
            failures.append(
                "requires SM90+ and is not supported on the SM11x/SM12x families"
            )
        return failures

    @staticmethod
    def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
        """Per-row exclusive end offsets (int32, (num_rows,)) for top-k
        kernels that take ragged lengths.

        seq_lens is (B, next_n) per-row effective lens for native spec decode
        and (B, 1) otherwise, in which case per-row lens are derived the same
        way as the other decode top-k kernels.
        """
        if seq_lens.numel() == num_rows:
            row_ends = seq_lens.reshape(-1)
        else:
            next_n_offsets = torch.arange(
                next_n, dtype=torch.int32, device=seq_lens.device
            )
            row_ends = (
                (seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
                .clamp_(min=0)
                .reshape(-1)
            )
        assert row_ends.dtype == torch.int32
        return row_ends

    def forward(
        self,
        logits: torch.Tensor,
        seq_lens: torch.Tensor,
        next_n: int,
        topk_indices: torch.Tensor,
        topk_tokens: int,
        max_seq_len: int,
    ) -> None:
        """Run the resolved decode top-k implementation, writing into
        topk_indices (int32, -1 fill for rows shorter than topk_tokens).
        """
        backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
        logger.info_once("Sparse indexer decode top-k backend: %s", backend)
        if backend == "deep_select":
            row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
            deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
        elif backend == "cooperative":
            (topk_workspace,) = current_workspace_manager().get_simultaneous(
                ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
            )
            torch.ops._C.cooperative_topk(
                logits,
                self._row_ends(seq_lens, next_n, logits.shape[0]),
                topk_indices,
                topk_workspace,
                topk_tokens,
                max_seq_len,
            )
        elif backend == "persistent":
            (topk_workspace,) = current_workspace_manager().get_simultaneous(
                ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
            )
            # persistent_topk's last argument is the host-side "max seq_len
            # across rows" bound (it gates the sampled_topk path and clamps
            # per-row lengths), not the padded row width: logits is sized to
            # max_model_len, so passing its width would incorrectly enable the
            # long-row kernels on short-context batches.
            torch.ops._C.persistent_topk(
                logits,
                self._row_ends(seq_lens, next_n, logits.shape[0]),
                topk_indices,
                topk_workspace,
                topk_tokens,
                max_seq_len,
            )
        elif backend == "flashinfer":
            # Deferred: importing flashinfer initializes CUDA at import time.
            from flashinfer.topk import top_k_ragged_transform

            row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
            offsets = torch.zeros(
                logits.shape[0], dtype=torch.int32, device=logits.device
            )
            # top_k_ragged_transform selects within [0, row_ends[i]) per
            # row and -1-fills past the row length.
            indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
            topk_indices.copy_(indices)
        elif backend == "flashinfer_gvr2":
            # The production varlen entry point; the public top_k_varlen
            # wrapper adds ~50us of host-side validation per call, which the
            # eager indexer path pays on every decode step.
            from flashinfer.topk_varlen.kernels.gvr2_topk_host import run_varlen

            if not torch.cuda.is_current_stream_capturing():
                self._gvr2_warmup(logits, topk_tokens)
            # Every row is its own request (next_n=1) with the per-row
            # length as its kv length, so both seq_lens layouts map onto the
            # kernel's contract; compress_ratio stays 1 because the logits
            # and seq_lens are already in the same (possibly pooled) units.
            # max_seq_len=row width keeps the call free of host syncs. The
            # kernel needs a contiguous, 16B-aligned output and writes the
            # -1 fill past a short row itself.
            num_rows = logits.shape[0]
            row_ends = self._row_ends(seq_lens, next_n, num_rows).contiguous()
            direct = topk_indices.is_contiguous() and topk_indices.data_ptr() % 16 == 0
            out = topk_indices if direct else torch.empty_like(topk_indices)
            run_varlen(
                logits,
                None,
                row_ends,
                out,
                next_n=1,
                compress_ratio=1,
                max_seq_len=logits.shape[1],
                top_k=topk_tokens,
            )
            if not direct:
                topk_indices.copy_(out)
        elif backend == "torch":
            # Debug reference: mask everything past each row's end, then topk.
            row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
            cols = torch.arange(logits.shape[1], device=logits.device)
            masked = logits.masked_fill(
                cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
            )
            indices = masked.topk(topk_tokens, dim=-1).indices
            in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
                0
            ) < row_ends.unsqueeze(1)
            indices = torch.where(in_range, indices, -1)
            topk_indices.copy_(indices)
        elif backend == "aiter":
            from vllm._aiter_ops import rocm_aiter_ops

            # AITER's decode kernel only implements the 1D/next_n == 1 form,
            # which is equivalent once the ragged ends are expanded.
            rocm_aiter_ops.indexer_top_k_decode(
                logits,
                1,
                self._row_ends(seq_lens, next_n, logits.shape[0]),
                topk_indices,
                topk_tokens,
            )
        else:
            assert backend == "per_row", f"unknown topk backend: {backend}"
            self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)

    @staticmethod
    def _per_row(
        logits: torch.Tensor,
        seq_lens: torch.Tensor,
        next_n: int,
        topk_indices: torch.Tensor,
        topk_tokens: int,
    ) -> None:
        ops.top_k_per_row_decode(
            logits,
            next_n,
            seq_lens,
            topk_indices,
            logits.shape[0],
            logits.stride(0),
            logits.stride(1),
            topk_tokens,
        )

_cooperative_constraints(logits, topk_tokens, num_rows)

Unmet constraints of cooperative_topk (empty when applicable).

Source code in vllm/model_executor/layers/indexer_topk.py
def _cooperative_constraints(
    self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> list[str]:
    """Unmet constraints of cooperative_topk (empty when applicable)."""
    failures = []
    if not self._is_cuda:
        failures.append("requires a CUDA platform")
    if topk_tokens not in (512, 1024, 2048):
        failures.append(
            f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
        )
    if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
        failures.append(
            f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
        )
    if logits.dtype not in (torch.float32, torch.bfloat16):
        failures.append(f"logits must be fp32 or bf16, got {logits.dtype}")
    # Rows are TMA-loaded in 16-byte units.
    align = 16 // logits.element_size()
    if logits.stride(0) % align != 0:
        failures.append(
            f"logits.stride(0) must be divisible by {align}, got {logits.stride(0)}"
        )
    if self._is_cuda and not self._cooperative_capable:
        failures.append(
            "requires SM90+ and is not supported on the SM11x/SM12x families"
        )
    return failures

_gvr2_constraints(logits, topk_tokens) staticmethod

Unmet input constraints of gvr_2 (empty when applicable).

Source code in vllm/model_executor/layers/indexer_topk.py
@staticmethod
def _gvr2_constraints(logits: torch.Tensor, topk_tokens: int) -> list[str]:
    """Unmet input constraints of gvr_2 (empty when applicable)."""
    failures = []
    if topk_tokens not in (512, 1024, 2048):
        failures.append(
            f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
        )
    if logits.dtype != torch.float32 or logits.stride(1) != 1:
        failures.append(
            f"requires fp32 logits with stride(1) == 1, got "
            f"dtype={logits.dtype}, stride={logits.stride()}"
        )
    # float4 row loads: the row pitch must be a multiple of 4 elements and
    # the base 16-byte aligned; rows may not overlap (an expand view).
    num_rows, width = logits.shape
    row_pitch = logits.stride(0) if num_rows > 1 else width
    if row_pitch % 4 != 0 or (num_rows > 1 and row_pitch < width):
        failures.append(
            f"logits row stride must be a multiple of 4 and at least the "
            f"row width, got stride(0)={row_pitch}, width={width}"
        )
    elif num_rows > 1 and logits.untyped_storage().nbytes() < (
        (logits.storage_offset() + num_rows * row_pitch) * logits.element_size()
    ):
        failures.append("logits storage must cover every row's full stride")
    if logits.data_ptr() % 16 != 0:
        failures.append("logits base pointer must be 16-byte aligned")
    return failures

_gvr2_warmup(logits, topk_tokens)

Compile gvr_2's kernel variants for every decode row count once.

The kernel family and template depend on the row count, and the indexer runs eagerly (outside CUDA graphs) in some models, so without this every new batch size would JIT-compile (~1 s) in the middle of serving. Must run eagerly; the launchers are cached per row count.

Source code in vllm/model_executor/layers/indexer_topk.py
def _gvr2_warmup(self, logits: torch.Tensor, topk_tokens: int) -> None:
    """Compile gvr_2's kernel variants for every decode row count once.

    The kernel family and template depend on the row count, and the
    indexer runs eagerly (outside CUDA graphs) in some models, so without
    this every new batch size would JIT-compile (~1 s) in the middle of
    serving. Must run eagerly; the launchers are cached per row count.
    """
    key = (logits.shape[1], logits.stride(0), topk_tokens)
    if key in self._gvr2_warmed:
        return
    from flashinfer.topk_varlen.kernels.gvr2_topk_host import warmup_varlen

    # warmup_varlen allocates rows x stride fp32 once; keep it under 256MB.
    rows_cap = max(1, (256 << 20) // (logits.stride(0) * 4))
    max_rows = max(logits.shape[0], min(self._gvr2_max_rows, rows_cap))
    start = time.perf_counter()
    warmup_varlen(
        top_k=topk_tokens,
        max_seq_len=logits.shape[1],
        compress_ratio=1,
        next_n=1,
        num_rows_list=range(1, max_rows + 1),
        row_stride=logits.stride(0),
    )
    self._gvr2_warmed.add(key)
    logger.info_once(
        "Warmed up gvr_2 top-k for up to %d rows x %d (topk %d) in %.1fs",
        max_rows,
        logits.shape[1],
        topk_tokens,
        time.perf_counter() - start,
    )

_resolve_auto(logits, topk_tokens, num_rows)

The priority chain: cooperative -> persistent -> per_row. aiter/deep_select/flashinfer/flashinfer_gvr2/torch are opt-in only.

cooperative_topk is preferred whenever it is applicable, i.e. within its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to persistent_topk.

aiter not auto-resolved as it does not outperform the per-row on every shape, hence we delegate to the caller to decide when to use it.

Source code in vllm/model_executor/layers/indexer_topk.py
def _resolve_auto(
    self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
    """The priority chain: cooperative -> persistent -> per_row.
    aiter/deep_select/flashinfer/flashinfer_gvr2/torch are opt-in only.

    cooperative_topk is preferred whenever it is applicable, i.e. within
    its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
    persistent_topk.

    aiter not auto-resolved as it does not outperform the per-row on every shape,
    hence we delegate to the caller to decide when to use it.
    """
    if not self._cooperative_constraints(logits, topk_tokens, num_rows):
        return "cooperative"
    if self._is_cuda and topk_tokens in (512, 1024, 2048):
        return "persistent"
    return "per_row"

_row_ends(seq_lens, next_n, num_rows) staticmethod

Per-row exclusive end offsets (int32, (num_rows,)) for top-k kernels that take ragged lengths.

seq_lens is (B, next_n) per-row effective lens for native spec decode and (B, 1) otherwise, in which case per-row lens are derived the same way as the other decode top-k kernels.

Source code in vllm/model_executor/layers/indexer_topk.py
@staticmethod
def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
    """Per-row exclusive end offsets (int32, (num_rows,)) for top-k
    kernels that take ragged lengths.

    seq_lens is (B, next_n) per-row effective lens for native spec decode
    and (B, 1) otherwise, in which case per-row lens are derived the same
    way as the other decode top-k kernels.
    """
    if seq_lens.numel() == num_rows:
        row_ends = seq_lens.reshape(-1)
    else:
        next_n_offsets = torch.arange(
            next_n, dtype=torch.int32, device=seq_lens.device
        )
        row_ends = (
            (seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
            .clamp_(min=0)
            .reshape(-1)
        )
    assert row_ends.dtype == torch.int32
    return row_ends

forward(logits, seq_lens, next_n, topk_indices, topk_tokens, max_seq_len)

Run the resolved decode top-k implementation, writing into topk_indices (int32, -1 fill for rows shorter than topk_tokens).

Source code in vllm/model_executor/layers/indexer_topk.py
def forward(
    self,
    logits: torch.Tensor,
    seq_lens: torch.Tensor,
    next_n: int,
    topk_indices: torch.Tensor,
    topk_tokens: int,
    max_seq_len: int,
) -> None:
    """Run the resolved decode top-k implementation, writing into
    topk_indices (int32, -1 fill for rows shorter than topk_tokens).
    """
    backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
    logger.info_once("Sparse indexer decode top-k backend: %s", backend)
    if backend == "deep_select":
        row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
        deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
    elif backend == "cooperative":
        (topk_workspace,) = current_workspace_manager().get_simultaneous(
            ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
        )
        torch.ops._C.cooperative_topk(
            logits,
            self._row_ends(seq_lens, next_n, logits.shape[0]),
            topk_indices,
            topk_workspace,
            topk_tokens,
            max_seq_len,
        )
    elif backend == "persistent":
        (topk_workspace,) = current_workspace_manager().get_simultaneous(
            ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
        )
        # persistent_topk's last argument is the host-side "max seq_len
        # across rows" bound (it gates the sampled_topk path and clamps
        # per-row lengths), not the padded row width: logits is sized to
        # max_model_len, so passing its width would incorrectly enable the
        # long-row kernels on short-context batches.
        torch.ops._C.persistent_topk(
            logits,
            self._row_ends(seq_lens, next_n, logits.shape[0]),
            topk_indices,
            topk_workspace,
            topk_tokens,
            max_seq_len,
        )
    elif backend == "flashinfer":
        # Deferred: importing flashinfer initializes CUDA at import time.
        from flashinfer.topk import top_k_ragged_transform

        row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
        offsets = torch.zeros(
            logits.shape[0], dtype=torch.int32, device=logits.device
        )
        # top_k_ragged_transform selects within [0, row_ends[i]) per
        # row and -1-fills past the row length.
        indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
        topk_indices.copy_(indices)
    elif backend == "flashinfer_gvr2":
        # The production varlen entry point; the public top_k_varlen
        # wrapper adds ~50us of host-side validation per call, which the
        # eager indexer path pays on every decode step.
        from flashinfer.topk_varlen.kernels.gvr2_topk_host import run_varlen

        if not torch.cuda.is_current_stream_capturing():
            self._gvr2_warmup(logits, topk_tokens)
        # Every row is its own request (next_n=1) with the per-row
        # length as its kv length, so both seq_lens layouts map onto the
        # kernel's contract; compress_ratio stays 1 because the logits
        # and seq_lens are already in the same (possibly pooled) units.
        # max_seq_len=row width keeps the call free of host syncs. The
        # kernel needs a contiguous, 16B-aligned output and writes the
        # -1 fill past a short row itself.
        num_rows = logits.shape[0]
        row_ends = self._row_ends(seq_lens, next_n, num_rows).contiguous()
        direct = topk_indices.is_contiguous() and topk_indices.data_ptr() % 16 == 0
        out = topk_indices if direct else torch.empty_like(topk_indices)
        run_varlen(
            logits,
            None,
            row_ends,
            out,
            next_n=1,
            compress_ratio=1,
            max_seq_len=logits.shape[1],
            top_k=topk_tokens,
        )
        if not direct:
            topk_indices.copy_(out)
    elif backend == "torch":
        # Debug reference: mask everything past each row's end, then topk.
        row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
        cols = torch.arange(logits.shape[1], device=logits.device)
        masked = logits.masked_fill(
            cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
        )
        indices = masked.topk(topk_tokens, dim=-1).indices
        in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
            0
        ) < row_ends.unsqueeze(1)
        indices = torch.where(in_range, indices, -1)
        topk_indices.copy_(indices)
    elif backend == "aiter":
        from vllm._aiter_ops import rocm_aiter_ops

        # AITER's decode kernel only implements the 1D/next_n == 1 form,
        # which is equivalent once the ragged ends are expanded.
        rocm_aiter_ops.indexer_top_k_decode(
            logits,
            1,
            self._row_ends(seq_lens, next_n, logits.shape[0]),
            topk_indices,
            topk_tokens,
        )
    else:
        assert backend == "per_row", f"unknown topk backend: {backend}"
        self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)

forward_prefill(logits, row_starts, row_ends, topk_indices, topk_tokens)

Prefill top-k: row r selects over logits[r, row_starts[r]:row_ends[r]] (row_starts None means 0) and writes indices relative to the row start (int32, -1 fill for rows shorter than topk_tokens).

Source code in vllm/model_executor/layers/indexer_topk.py
def forward_prefill(
    self,
    logits: torch.Tensor,
    row_starts: torch.Tensor | None,
    row_ends: torch.Tensor,
    topk_indices: torch.Tensor,
    topk_tokens: int,
) -> None:
    """Prefill top-k: row `r` selects over
    `logits[r, row_starts[r]:row_ends[r]]` (row_starts None means 0) and
    writes indices relative to the row start (int32, -1 fill for rows
    shorter than topk_tokens)."""
    num_rows = logits.shape[0]
    backend = self.resolve_prefill_backend(logits, topk_tokens, row_starts)
    if backend == "flashinfer_gvr2":
        logger.info_once("Sparse indexer prefill top-k backend: flashinfer_gvr2")
        from flashinfer.topk_varlen.kernels.gvr2_topk_host import run_varlen

        direct = topk_indices.is_contiguous() and topk_indices.data_ptr() % 16 == 0
        out = topk_indices if direct else torch.empty_like(topk_indices)
        # max_seq_len=row width avoids the host read of row_ends.max().
        run_varlen(
            logits,
            None,
            row_ends.contiguous(),
            out,
            next_n=1,
            compress_ratio=1,
            max_seq_len=logits.shape[1],
            top_k=topk_tokens,
        )
        if not direct:
            topk_indices.copy_(out)
        return
    ops.top_k_per_row_prefill(
        logits,
        torch.zeros_like(row_ends) if row_starts is None else row_starts,
        row_ends,
        topk_indices,
        num_rows,
        logits.stride(0),
        logits.stride(1),
        topk_tokens,
    )

resolve_backend(logits, topk_tokens, num_rows)

Resolve the decode top-k implementation from the configured backend ("auto" = the pre-existing chain, or a validated explicit value).

Source code in vllm/model_executor/layers/indexer_topk.py
def resolve_backend(
    self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
    """Resolve the decode top-k implementation from the configured
    backend ("auto" = the pre-existing chain, or a validated explicit
    value)."""
    if self._backend == "auto":
        return self._resolve_auto(logits, topk_tokens, num_rows)

    failures: list[str] = []
    if self._backend == "cooperative":
        failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
    elif self._backend == "persistent":
        if not (self._is_cuda or self._is_rocm):
            failures.append("requires a CUDA or ROCm platform")
        if topk_tokens not in (512, 1024, 2048):
            failures.append(
                f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
            )
    elif self._backend == "deep_select":
        if not self._is_cuda:
            failures.append("requires a CUDA platform")
        elif not self._has_deep_select:
            failures.append("requires SM100a/SM103a (10.x device family)")
        elif not _HAS_DEEP_SELECT_C:
            failures.append("vllm._deepselect_C could not be imported")
        elif not is_deep_select_supported(logits, topk_tokens):
            failures.append(
                f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
                f" stride={logits.stride()}, topk={topk_tokens}"
            )
    elif self._backend == "aiter":
        if not self._is_rocm:
            failures.append("requires a ROCm platform")
        elif not self._aiter_enabled:
            failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
        elif not self._aiter_capable:
            failures.append(
                "AITER has no tuned indexer top-k kernels for this GPU arch"
            )
        if topk_tokens not in (512, 1024, 2048, 4096):
            failures.append(
                f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
            )
    elif self._backend == "flashinfer":
        if not self._is_cuda:
            failures.append("requires a CUDA platform")
        if not self._has_flashinfer_topk:
            failures.append(
                "flashinfer.topk.top_k_ragged_transform is not importable"
            )
        if logits.dtype != torch.float32 or logits.stride(1) != 1:
            failures.append(
                f"requires fp32 logits with stride(1) == 1, got "
                f"dtype={logits.dtype}, stride={logits.stride()}"
            )
    elif self._backend == "flashinfer_gvr2":
        if not self._is_cuda:
            failures.append("requires a CUDA platform")
        elif not current_platform.is_device_capability_family(100):
            failures.append("requires the SM100 device family")
        elif not self._has_flashinfer_gvr2:
            failures.append("flashinfer.top_k_varlen has no gvr_2 backend")
        failures += self._gvr2_constraints(logits, topk_tokens)
    if failures:
        raise RuntimeError(
            f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
            + "; ".join(failures)
        )
    return self._backend

resolve_prefill_backend(logits, topk_tokens, row_starts)

flashinfer_gvr2 for chunks of at least GVR2_PREFILL_MIN_ROWS row-relative rows (row_starts is None) when it was requested explicitly, otherwise per_row (top_k_per_row_prefill).

gvr_2 reads every row from column 0. Unmet gvr_2 constraints fall back instead of raising, since the same backend setting also drives decode.

Source code in vllm/model_executor/layers/indexer_topk.py
def resolve_prefill_backend(
    self,
    logits: torch.Tensor,
    topk_tokens: int,
    row_starts: torch.Tensor | None,
) -> str:
    """flashinfer_gvr2 for chunks of at least GVR2_PREFILL_MIN_ROWS
    row-relative rows (row_starts is None) when it was requested
    explicitly, otherwise per_row (top_k_per_row_prefill).

    gvr_2 reads every row from column 0. Unmet gvr_2 constraints fall
    back instead of raising, since the same backend setting also drives
    decode.
    """
    if (
        self._backend == "flashinfer_gvr2"
        and row_starts is None
        and logits.shape[0] >= GVR2_PREFILL_MIN_ROWS
        and self._has_flashinfer_gvr2
        and not self._gvr2_constraints(logits, topk_tokens)
    ):
        return "flashinfer_gvr2"
    return "per_row"

takes_bf16_logits(logits, topk_tokens, num_rows)

Whether this decode call can consume BF16 without a full FP32 copy.

Source code in vllm/model_executor/layers/indexer_topk.py
def takes_bf16_logits(
    self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> bool:
    """Whether this decode call can consume BF16 without a full FP32 copy."""
    if logits.dtype != torch.bfloat16 or logits.stride(1) != 1:
        return False
    if self._backend == "deep_select":
        return (
            self._has_deep_select
            and _HAS_DEEP_SELECT_C
            and is_deep_select_supported(logits, topk_tokens)
        )
    if self._backend == "cooperative":
        return not self._cooperative_constraints(logits, topk_tokens, num_rows)
    if (
        self._backend == "auto"
        and self._resolve_auto(logits, topk_tokens, num_rows) == "cooperative"
    ):
        return True
    return (
        self._backend in ("auto", "persistent")
        and self._is_cuda
        and current_platform.is_device_capability_family(100)
        and topk_tokens in (512, 1024, 2048)
        and self.resolve_backend(logits, topk_tokens, num_rows) == "persistent"
    )

_get_empty_and_aligned_tensor(dim0, dim1, device, dtype)

Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned.

Source code in vllm/model_executor/layers/indexer_topk.py
def _get_empty_and_aligned_tensor(
    dim0: int, dim1: int, device: torch.device, dtype: torch.dtype
) -> torch.Tensor:
    """Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned."""
    output_stride_requirement = get_deep_select_stride_requirement()[1] // (
        dtype.itemsize
    )
    assert output_stride_requirement > 0
    dim1_rounded = (
        (dim1 + output_stride_requirement - 1) // output_stride_requirement
    ) * output_stride_requirement
    return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1]

deep_select_topk(input, topk, end=None, output_idx=None, indices_dtype=torch.int32)

Select the top-k indices per row of input with DeepSelect.

Parameters:

  • input

    (Tensor) –

    (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and stride(0) must be 1024B-aligned.

  • topk

    (int) –

    Number of elements to select per row; must be <= 4096.

  • end

    (Tensor | None, default: None ) –

    Optional (num_rows,) int32 tensor with the exclusive right boundary of each row. Rows with end[i] < topk get their remaining indices filled with -1.

  • output_idx

    (Tensor | None, default: None ) –

    Optional preallocated (num_rows, topk) output tensor whose stride(0) is 32B-aligned (e.g. a slice of a wider buffer).

  • indices_dtype

    (dtype, default: int32 ) –

    Output dtype when output_idx is not provided.

Returns:

  • Tensor –

    The (num_rows, topk) indices tensor.

Source code in vllm/model_executor/layers/indexer_topk.py
def deep_select_topk(
    input: torch.Tensor,
    topk: int,
    end: torch.Tensor | None = None,
    output_idx: torch.Tensor | None = None,
    indices_dtype: torch.dtype = torch.int32,
) -> torch.Tensor:
    """Select the top-k indices per row of `input` with DeepSelect.

    Args:
        input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and
            stride(0) must be 1024B-aligned.
        topk: Number of elements to select per row; must be <= 4096.
        end: Optional (num_rows,) int32 tensor with the exclusive right
            boundary of each row. Rows with `end[i] < topk` get their
            remaining indices filled with -1.
        output_idx: Optional preallocated (num_rows, topk) output tensor whose
            stride(0) is 32B-aligned (e.g. a slice of a wider buffer).
        indices_dtype: Output dtype when `output_idx` is not provided.

    Returns:
        The (num_rows, topk) indices tensor.

    """
    assert input.dim() == 2 and input.stride(1) == 1
    assert (
        input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0]
        == 0
    )

    num_rows = input.shape[0]
    if output_idx is None:
        output_idx = _get_empty_and_aligned_tensor(
            num_rows, topk, input.device, indices_dtype
        )
    else:
        assert output_idx.dtype == indices_dtype
        assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk

    torch.ops.deep_select.topk(
        input,
        topk,
        None,  # begin is not supported
        end,
        False,  # sorted_value
        False,  # sorted_index
        None,  # output_value
        output_idx,
        None,  # output_idx_offset
        IDX_OOB_FILL_VALUE,
        float("-inf"),  # value_oob_fill_value
        False,  # return_value
        False,  # abort_when_nan_found: dummy/capture passes can feed NaN
        # logits from uninitialized KV cache; the reference topk kernels do
        # not trap on NaN, and trapping would abort CUDA graph capture.
    )
    return output_idx

get_deep_select_stride_requirement() cached

Stride alignment requirement (input, output) in bytes.

Source code in vllm/model_executor/layers/indexer_topk.py
@functools.lru_cache(maxsize=1)
def get_deep_select_stride_requirement() -> tuple[int, int]:
    """Stride alignment requirement (input, output) in bytes."""
    return torch.ops.deep_select.get_alignment_requirement()

is_deep_select_supported(input, topk)

Whether the kernel accepts this input (dtype/stride/topk constraints).

Source code in vllm/model_executor/layers/indexer_topk.py
def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool:
    """Whether the kernel accepts this input (dtype/stride/topk constraints)."""
    return (
        topk <= 4096
        and input.shape[1] < 2**23
        and input.dtype in (torch.float32, torch.bfloat16)
        and input.stride(1) == 1
        and input.stride(0)
        * input.element_size()
        % get_deep_select_stride_requirement()[0]
        == 0
    )