Skip to content

vllm.model_executor.models.diffusion_gemma_sampler

Triton kernels for the DiffusionGemma denoise sampler.

The step needs, for every canvas position, the argmax of the temperature-scaled logits, a Gumbel-max sample from them, the entropy of their softmax, and the softmax itself in the model dtype for the self-conditioning matmul. As PyTorch ops these are several passes over a [rows, vocab] fp32 tensor plus a same-sized noise tensor. The Triton kernel reads each row once with an online max and sum, draws the noise inline, and writes the probabilities once.

sample_step then updates each request's canvas and convergence state from those statistics, and softcap_logits applies the final logit softcap in one pass. Each has a PyTorch reference for tests and non-CUDA runs. The dummy runs never take a decode step, so the model registers these kernels for JIT warmup with every variant serving reaches.

The logprobs of a converging slot are written to a per-slot stash by stash_converged_logprobs, which skips the rows of slots that did not converge.

Functions:

_empty(like, *shape, dtype)

torch.empty on like's device, or only its metadata when like is a warmup input, so warmup allocates nothing.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def _empty(like: Any, *shape: int, dtype: torch.dtype) -> Any:
    """`torch.empty` on `like`'s device, or only its metadata when
    `like` is a warmup input, so warmup allocates nothing."""
    if isinstance(like, TritonWarmupTensor):
        return _warmup_tensor(dtype, *shape)
    return torch.empty(shape, dtype=dtype, device=like.device)

_lp_topk_kernel(logits_ptr, logits_stride, src_ptr, keep_ptr, valid_ptr, temps_ptr, slots_ptr, num_ids_ptr, block_max_ptr, cand_ptr, K, W, V, NUM_CHUNKS, NUM_BLOCKS, HAS_IDS, BLOCK, BLOCKS_PER_CHUNK, BLOCKS_POW2, KP)

The KP largest keys of one chunk of a row, unordered.

A key orders by value, then by lower index: the fp32 bits mapped to an order-preserving int32 in the high word, the inverted index in the low. Keys are distinct, so the top-k under them is unique.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
@triton.jit
def _lp_topk_kernel(
    logits_ptr,
    logits_stride,
    src_ptr,
    keep_ptr,
    valid_ptr,
    temps_ptr,
    slots_ptr,
    num_ids_ptr,
    block_max_ptr,
    cand_ptr,
    K,
    W,
    V,
    NUM_CHUNKS,
    NUM_BLOCKS,
    HAS_IDS: tl.constexpr,
    BLOCK: tl.constexpr,
    BLOCKS_PER_CHUNK: tl.constexpr,
    BLOCKS_POW2: tl.constexpr,
    KP: tl.constexpr,
):
    """The KP largest keys of one chunk of a row, unordered.

    A key orders by value, then by lower index: the fp32 bits mapped to an
    order-preserving int32 in the high word, the inverted index in the low.
    Keys are distinct, so the top-k under them is unique.
    """
    j = tl.program_id(0)
    pos = tl.program_id(1)
    chunk = tl.program_id(2)
    if _lp_skip_topk(keep_ptr, valid_ptr, slots_ptr, num_ids_ptr, j, pos, HAS_IDS):
        return
    r = (j * W + pos).to(tl.int64)
    # K block maxima are K distinct elements at or above the K-th largest
    # block max, so every top-k element is too: the rest of the row is
    # skipped without a reduction over it.
    b = tl.arange(0, BLOCKS_POW2)
    block_max = tl.load(
        block_max_ptr + r * NUM_BLOCKS + b, mask=b < NUM_BLOCKS, other=float("-inf")
    )
    sorted_max = tl.sort(_lp_key32(block_max), descending=True)
    kth = tl.max(tl.where(b == K - 1, sorted_max, -0x7FFFFFFF - 1), 0)
    threshold = kth.to(tl.int64) << 32
    # Distinct placeholders below every real key.
    best = _NO_KEY + 1 + tl.arange(0, KP).to(tl.int64)
    best_min = tl.min(best, 0)
    for blk in range(BLOCKS_PER_CHUNK):
        offs = (chunk * BLOCKS_PER_CHUNK + blk) * BLOCK + tl.arange(0, BLOCK)
        x = _lp_load(
            logits_ptr,
            logits_stride,
            src_ptr,
            temps_ptr,
            j,
            pos,
            W,
            offs,
            V,
            float("-inf"),
        )
        key = (_lp_key32(x).to(tl.int64) << 32) | (0x7FFFFFFF - offs).to(tl.int64)
        key = tl.where((offs < V) & (key >= threshold), key, _NO_KEY)
        key_max = tl.max(key, 0)
        while key_max > best_min:
            best = tl.where(best == best_min, key_max, best)
            best_min = tl.min(best, 0)
            key = tl.where(key == key_max, _NO_KEY, key)
            key_max = tl.max(key, 0)
    out = (r * NUM_CHUNKS + chunk) * KP + tl.arange(0, KP)
    tl.store(cand_ptr + out, best)

_tile_rows(max_tile_rows, cols, *itemsizes)

Row counts for a [rows, cols] tile reaching each storage-size class of tensors with these element sizes.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def _tile_rows(max_tile_rows: int, cols: int, *itemsizes: int) -> list[int]:
    """Row counts for a `[rows, cols]` tile reaching each storage-size class
    of tensors with these element sizes."""
    rows = {2}
    for itemsize in itemsizes:
        large = _MAX_32BIT_BYTES // (cols * itemsize) + 1
        if large <= max_tile_rows:
            rows.add(large)
    return sorted(rows)

sample_row_stats(logits, temps, canvas_len, seed, probs_dtype)

Argmax, Gumbel-max sample, entropy and (optionally) softmax per row.

logits is [rows, vocab] and row i uses temperature temps[i // canvas_len]. A zero temperature is greedy: the sample is the argmax, and the entropy is the reference's, which clamps the temperature at 1e-10.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
@triton_kernel_dispatcher_with_warmup(
    kernel=_row_stats_kernel, warmup_inputs=_row_stats_warmup_inputs
)
def sample_row_stats(
    logits: torch.Tensor,
    temps: torch.Tensor,
    canvas_len: int,
    seed: int,
    probs_dtype: torch.dtype | None,
) -> DispatchSpec:
    """Argmax, Gumbel-max sample, entropy and (optionally) softmax per row.

    `logits` is `[rows, vocab]` and row `i` uses temperature
    `temps[i // canvas_len]`. A zero temperature is greedy: the sample is
    the argmax, and the entropy is the reference's, which clamps the
    temperature at 1e-10.
    """
    rows, vocab = logits.shape
    argmax = _empty(logits, rows, dtype=torch.int64)
    sample = _empty(logits, rows, dtype=torch.int64)
    entropy = _empty(logits, rows, dtype=torch.float32)
    if probs_dtype is not None:
        probs = _empty(logits, rows, vocab, dtype=probs_dtype)
        probs_arg, probs_stride = probs, probs.stride(0)
    else:
        probs = None
        probs_arg, probs_stride = entropy, 0
    outputs = argmax, sample, entropy, probs
    return (
        (rows,),
        dict(
            logits_ptr=logits,
            logits_stride=logits.stride(0),
            temps_ptr=temps,
            argmax_ptr=argmax,
            sample_ptr=sample,
            entropy_ptr=entropy,
            probs_ptr=probs_arg,
            probs_stride=probs_stride,
            seed=seed,
            V=vocab,
            CL=canvas_len,
            WRITE_PROBS=probs is not None,
            BLOCK=4096,
            num_warps=8,
        ),
        outputs,
    )

sample_row_stats_reference(logits, temps, canvas_len, probs_dtype)

The PyTorch form of the same statistics, for tests and non-CUDA runs.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def sample_row_stats_reference(
    logits: torch.Tensor,
    temps: torch.Tensor,
    canvas_len: int,
    probs_dtype: torch.dtype | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
    """The PyTorch form of the same statistics, for tests and non-CUDA runs."""
    temp = temps.repeat_interleave(canvas_len).float()[:, None]
    scaled = logits.float() / temp.clamp(min=1e-10)
    u = torch.rand_like(scaled).clamp(min=1e-20)
    noisy = scaled + (-torch.log(-torch.log(u))) * (temp > 0).float()
    log_probs = scaled.log_softmax(dim=-1)
    probs = log_probs.exp()
    # Masked (-inf) columns: 0 * -inf is NaN, so they are dropped from the sum.
    entropy = -torch.where(probs > 0, probs * log_probs, 0.0).sum(dim=-1)
    out_probs = probs.to(probs_dtype) if probs_dtype is not None else None
    return scaled.argmax(dim=-1), noisy.argmax(dim=-1), entropy, out_probs

sample_step(*, new_tokens, argmax_tokens, token_entropy, random_tokens, decode_slots, decode_idx, valid_canvas_len, canvas, argmax_canvas, history, pin_mask, seed_canvas, step, is_encoder_phase, confident, history_len, max_steps, read_only, sampled, num_sampled, confidence_threshold, entropy_bound)

Advance each decode request one step: confidence, entropy-bound accept/renoise, the history ring and convergence.

The contiguous [n, W] statistics come from sample_row_stats and random_tokens is the renoise draw. Per-slot state is indexed by decode_slots and updated in place over its first W columns; sampled and num_sampled are written at decode_idx. Returns which requests denoise again and so read their soft embed next step.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
@triton_kernel_dispatcher_with_warmup(
    kernel=_sample_step_kernel, warmup_inputs=_sample_step_warmup_inputs
)
def sample_step(
    *,
    new_tokens: torch.Tensor,
    argmax_tokens: torch.Tensor,
    token_entropy: torch.Tensor,
    random_tokens: torch.Tensor,
    decode_slots: torch.Tensor,
    decode_idx: torch.Tensor,
    valid_canvas_len: torch.Tensor,
    canvas: torch.Tensor,
    argmax_canvas: torch.Tensor,
    history: torch.Tensor,
    pin_mask: torch.Tensor,
    seed_canvas: torch.Tensor,
    step: torch.Tensor,
    is_encoder_phase: torch.Tensor,
    confident: torch.Tensor,
    history_len: torch.Tensor,
    max_steps: torch.Tensor,
    read_only: torch.Tensor,
    sampled: torch.Tensor,
    num_sampled: torch.Tensor,
    confidence_threshold: float,
    entropy_bound: float,
) -> DispatchSpec:
    """Advance each decode request one step: confidence, entropy-bound
    accept/renoise, the history ring and convergence.

    The contiguous `[n, W]` statistics come from `sample_row_stats` and
    `random_tokens` is the renoise draw. Per-slot state is indexed by
    `decode_slots` and updated in place over its first `W` columns;
    `sampled` and `num_sampled` are written at `decode_idx`. Returns
    which requests denoise again and so read their soft embed next step.
    """
    n, W = new_tokens.shape
    for stats in (new_tokens, argmax_tokens, token_entropy, random_tokens):
        assert stats.shape == (n, W) and stats.stride() == (W, 1)
    for state in (canvas, argmax_canvas, history, pin_mask, seed_canvas, sampled):
        assert state.stride(-1) == 1
    keep = _empty(new_tokens, n, dtype=torch.bool)
    # Sized by the full canvas so narrow tiles share the compiled kernel.
    BLOCK = triton.next_power_of_2(canvas.shape[1])
    # The remaining tensors, strides and scalars bind to the kernel's
    # arguments by name.
    return (
        (n,),
        dict(
            keep_ptr=keep,
            W=W,
            ST=history.shape[1],
            BLOCK=BLOCK,
            num_warps=min(8, max(1, BLOCK // 256)),
        ),
        keep,
    )

sample_step_reference(*, new_tokens, argmax_tokens, token_entropy, random_tokens, decode_slots, decode_idx, valid_canvas_len, canvas, argmax_canvas, history, pin_mask, seed_canvas, step, is_encoder_phase, confident, history_len, max_steps, read_only, sampled, num_sampled, confidence_threshold, entropy_bound)

The PyTorch form of the same step, for tests and non-CUDA runs.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def sample_step_reference(
    *,
    new_tokens: torch.Tensor,
    argmax_tokens: torch.Tensor,
    token_entropy: torch.Tensor,
    random_tokens: torch.Tensor,
    decode_slots: torch.Tensor,
    decode_idx: torch.Tensor,
    valid_canvas_len: torch.Tensor,
    canvas: torch.Tensor,
    argmax_canvas: torch.Tensor,
    history: torch.Tensor,
    pin_mask: torch.Tensor,
    seed_canvas: torch.Tensor,
    step: torch.Tensor,
    is_encoder_phase: torch.Tensor,
    confident: torch.Tensor,
    history_len: torch.Tensor,
    max_steps: torch.Tensor,
    read_only: torch.Tensor,
    sampled: torch.Tensor,
    num_sampled: torch.Tensor,
    confidence_threshold: float,
    entropy_bound: float,
) -> torch.Tensor:
    """The PyTorch form of the same step, for tests and non-CUDA runs."""
    num_decode, W = new_tokens.shape
    ST = history.shape[1]
    canvas = canvas[:, :W]
    argmax_canvas = argmax_canvas[:, :W]
    history = history[:, :, :W]
    pin_mask = pin_mask[:, :W]
    seed_canvas = seed_canvas[:, :W]
    sampled = sampled[:, :W]

    # ---- Confidence ----
    # A canvas truncated near max_model_len is zero-padded up to W by the
    # caller; those padded rows are uniform (max entropy, argmax 0), so they
    # never trigger early convergence and are stable, and only the real
    # ``valid_canvas_len`` tokens are committed (num_sampled below).
    mean_entropy = token_entropy.mean(dim=-1)
    confident[decode_slots] = mean_entropy < confidence_threshold

    # ---- Entropy-bound acceptance mask ----
    sorted_ent, sorted_idx = torch.sort(token_entropy, dim=-1, stable=True)
    cumsum_ent = torch.cumsum(sorted_ent, dim=-1)
    cummax_ent = torch.cummax(sorted_ent, dim=-1).values
    sorted_mask = (cumsum_ent - cummax_ent) <= entropy_bound
    eb_mask = torch.zeros_like(sorted_mask)
    eb_mask.scatter_(1, sorted_idx, sorted_mask)

    # ---- Post-sample ----
    is_commit = is_encoder_phase[decode_slots]
    is_denoise = ~is_commit

    # Step update: +1 for denoise, reset to 0 for commit
    cur_step = step[decode_slots]
    step[decode_slots] = torch.where(
        is_denoise, cur_step + 1, cur_step.new_zeros(num_decode)
    )

    # Renoise draws from the whole vocabulary even for a constrained read.
    # The model was trained on random-vocabulary noise. Noise drawn from the
    # allowed set looks like garbled text, and later steps denoise it into
    # more garbled text.
    denoise_canvas = torch.where(eb_mask, new_tokens, random_tokens)
    denoise_canvas = torch.where(
        pin_mask[decode_slots], seed_canvas[decode_slots], denoise_canvas
    )
    canvas[decode_slots] = torch.where(
        is_commit.unsqueeze(1), random_tokens, denoise_canvas
    )

    # History: write argmax_tokens for denoise requests at circular position
    hist_len = history_len[decode_slots]
    write_pos = hist_len % ST
    for i in range(ST):
        write_here = ((write_pos == i) & is_denoise).unsqueeze(1)
        history[decode_slots, i] = torch.where(
            write_here, argmax_tokens, history[decode_slots, i]
        )

    # Argmax canvas: update for denoise, preserve for commit
    argmax_canvas[decode_slots] = torch.where(
        is_denoise.unsqueeze(1), argmax_tokens, argmax_canvas[decode_slots]
    )

    new_hist_len = torch.where(is_denoise, hist_len + 1, hist_len.new_zeros(num_decode))
    history_len[decode_slots] = new_hist_len

    # ---- Stability + convergence ----
    ref = history[decode_slots, 0]
    mismatch = torch.zeros(num_decode, device=new_tokens.device, dtype=torch.int32)
    for h in range(1, ST):
        mismatch = mismatch + (ref != history[decode_slots, h]).sum(dim=-1).int()
    stable = mismatch == 0

    converged = (stable & confident[decode_slots] & (new_hist_len >= ST)) | (
        step[decode_slots] >= max_steps[decode_slots]
    )
    # Commit done → denoise next (False); denoise converged → commit next (True)
    next_commit = is_denoise & converged
    keep = is_denoise & ~next_commit

    emit = is_commit | (next_commit & read_only[decode_slots])
    sampled[decode_idx] = argmax_canvas[decode_slots].to(sampled.dtype) * emit[:, None]
    num_sampled[decode_idx] = (emit * valid_canvas_len).to(num_sampled.dtype)

    # Overwrite canvas with argmax for newly converged denoise requests
    canvas[decode_slots] = torch.where(
        next_commit.unsqueeze(1), argmax_canvas[decode_slots], canvas[decode_slots]
    )
    is_encoder_phase[decode_slots] = next_commit & ~read_only[decode_slots]
    return keep

softcap_logits(logits, cap)

tanh(logits / cap) * cap in fp32, as a new [rows, vocab] tensor.

fp32 before tanh for numerical stability (matches HF DiffusionGemma).

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
@triton_kernel_dispatcher_with_warmup(
    kernel=_softcap_kernel, warmup_inputs=_softcap_warmup_inputs
)
def softcap_logits(logits: torch.Tensor, cap: float) -> DispatchSpec:
    """`tanh(logits / cap) * cap` in fp32, as a new `[rows, vocab]` tensor.

    fp32 before tanh for numerical stability (matches HF DiffusionGemma).
    """
    rows, vocab = logits.shape
    out = _empty(logits, rows, vocab, dtype=torch.float32)
    BLOCK = 4096
    return (
        (rows, triton.cdiv(vocab, BLOCK)),
        dict(
            logits_ptr=logits,
            logits_stride=logits.stride(0),
            out_ptr=out,
            out_stride=out.stride(0),
            V=vocab,
            cap=cap,
            BLOCK=BLOCK,
        ),
        out,
    )

stability_rows(generation_config)

Rows of the accepted-canvas history.

In Transformers, stability_threshold=1 means the current step must match the previous step. The history includes the current step, so it holds one more row.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def stability_rows(generation_config: dict[str, Any]) -> int:
    """Rows of the accepted-canvas history.

    In Transformers, `stability_threshold=1` means the current step must
    match the previous step. The history includes the current step, so it
    holds one more row.
    """
    return generation_config["stability_threshold"] + 1

stash_converged_logprobs(logits, canvas_len, src, slots, keep, valid, temps, num_topk, num_token_ids, token_ids, out_ids, out_logprobs, out_ranks, logits_mode)

Write the logprobs of the converged requests' rows to the stash.

Request j owns logits rows src[j] * canvas_len + p for p < valid[j], divided by temps[j]. Where keep[j], they go to out_*[slots[j], p], laid out as compute_topk_scores lays out a row: the argmax, then the request's token_ids if it has any, else the num_topk largest (tied ids by lower id), padded with token 0 at -inf. Rows not kept exit at once; the others are split into chunks so a few kept rows still fill the GPU.

Source code in vllm/model_executor/models/diffusion_gemma_sampler.py
def stash_converged_logprobs(
    logits: torch.Tensor,
    canvas_len: int,
    src: torch.Tensor,
    slots: torch.Tensor,
    keep: torch.Tensor,
    valid: torch.Tensor,
    temps: torch.Tensor,
    num_topk: int,
    num_token_ids: torch.Tensor,
    token_ids: torch.Tensor | None,
    out_ids: torch.Tensor,
    out_logprobs: torch.Tensor,
    out_ranks: torch.Tensor,
    logits_mode: bool,
) -> None:
    """Write the logprobs of the converged requests' rows to the stash.

    Request `j` owns logits rows `src[j] * canvas_len + p` for
    `p < valid[j]`, divided by `temps[j]`. Where `keep[j]`, they go to
    `out_*[slots[j], p]`, laid out as compute_topk_scores lays out a row:
    the argmax, then the request's `token_ids` if it has any, else the
    `num_topk` largest (tied ids by lower id), padded with token 0 at -inf.
    Rows not kept exit at once; the others are split into chunks so a few
    kept rows still fill the GPU.
    """
    m = slots.shape[0]
    vocab = logits.shape[-1]
    assert logits.stride(-1) == 1
    assert temps.dtype == torch.float32
    assert num_topk + 1 <= out_logprobs.shape[-1]
    block = _LP_BLOCK
    num_blocks = triton.cdiv(vocab, block)
    blocks_per_chunk = min(_LP_BLOCKS_PER_CHUNK, num_blocks)
    num_chunks = triton.cdiv(num_blocks, blocks_per_chunk)
    rows = m * canvas_len
    device = logits.device
    chunk_max = torch.empty(rows, num_chunks, dtype=torch.float32, device=device)
    chunk_arg = torch.empty(rows, num_chunks, dtype=torch.int64, device=device)
    block_sum = torch.empty(rows, num_blocks, dtype=torch.float32, device=device)
    block_cnt = torch.empty(rows, num_blocks, dtype=torch.int32, device=device)
    block_max = (
        torch.empty(rows, num_blocks, dtype=torch.float32, device=device)
        if num_topk > 0
        else chunk_max
    )
    has_ids = token_ids is not None
    ids_arg = token_ids if token_ids is not None else num_token_ids
    common = (logits, logits.stride(0), src, keep, valid, temps)
    grid = (m, canvas_len, num_chunks)
    _lp_max_kernel[grid](
        *common,
        chunk_max,
        chunk_arg,
        block_max,
        canvas_len,
        vocab,
        num_chunks,
        num_blocks,
        BLOCK=block,
        BLOCKS_PER_CHUNK=blocks_per_chunk,
        STORE_BLOCK_MAX=num_topk > 0,
        num_warps=4,
    )
    chunks_pow2 = triton.next_power_of_2(num_chunks)
    _lp_sum_kernel[grid](
        *common,
        chunk_max,
        block_sum,
        block_cnt,
        canvas_len,
        vocab,
        num_chunks,
        num_blocks,
        BLOCK=block,
        BLOCKS_PER_CHUNK=blocks_per_chunk,
        CHUNKS_POW2=chunks_pow2,
        num_warps=4,
    )
    topk = chunk_arg
    if num_topk > 0:
        kp = triton.next_power_of_2(num_topk)
        cand = torch.empty(rows, num_chunks, kp, dtype=torch.int64, device=device)
        _lp_topk_kernel[grid](
            *common,
            slots,
            num_token_ids,
            block_max,
            cand,
            num_topk,
            canvas_len,
            vocab,
            num_chunks,
            num_blocks,
            HAS_IDS=has_ids,
            BLOCK=block,
            BLOCKS_PER_CHUNK=blocks_per_chunk,
            BLOCKS_POW2=triton.next_power_of_2(max(num_blocks, num_topk)),
            KP=kp,
            num_warps=4,
        )
        topk = torch.empty(rows, num_topk, dtype=torch.int32, device=device)
        _lp_topk_merge_kernel[(m, canvas_len)](
            keep,
            valid,
            slots,
            num_token_ids,
            cand,
            topk,
            num_topk,
            canvas_len,
            num_chunks * kp,
            HAS_IDS=has_ids,
            CAND_POW2=triton.next_power_of_2(num_chunks * kp),
        )
    stash_width = out_logprobs.shape[-1]
    _lp_write_kernel[(m, canvas_len)](
        logits,
        logits.stride(0),
        src,
        slots,
        keep,
        valid,
        temps,
        chunk_max,
        chunk_arg,
        block_sum,
        block_cnt,
        topk,
        num_token_ids,
        ids_arg,
        ids_arg.stride(0) if has_ids else 0,
        out_ids,
        out_logprobs,
        out_ranks,
        num_topk,
        canvas_len,
        out_logprobs.shape[1],
        vocab,
        stash_width,
        num_chunks,
        num_blocks,
        HAS_IDS=has_ids,
        LOGITS_MODE=logits_mode,
        CHUNKS_POW2=chunks_pow2,
        BLOCKS_POW2=triton.next_power_of_2(num_blocks),
        SUM_GROUP=16,
        COL_BLOCK=min(triton.next_power_of_2(stash_width), 1024),
    )