Skip to content

vllm.distributed.kv_transfer.kv_connector.v1.offloading.state

Classes:

Functions:

RequestOffloadState dataclass

Methods:

  • storable_chunks –

    Number of allocated and keyed leading chunks eligible for store.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/state.py
@dataclass(slots=True)
class RequestOffloadState:
    config: SchedulerOffloadConfig
    req: Request
    req_context: ReqContext
    offloading_context: RequestOffloadingContext
    group_states: tuple[RequestGroupState, ...] = field(init=False)
    # upper bound on tokens to load for this request; None means no cap
    max_load_tokens: int | None = None
    # upper bound on tokens to offload for this request; None means no cap
    max_offload_tokens: int | None = None
    # number of hits in the GPU cache
    num_locally_computed_tokens: int = 0
    # In-flight job IDs. Per the connector's invariant, at any given time
    # this contains either a single load job, or one or more store jobs.
    transfer_jobs: set[int] = field(default_factory=set)
    # time.monotonic() of this request's first deferred offload lookup;
    # None once consumed (observed) or while no lookup is pending.
    deferred_lookup_start_time: float | None = None
    # Fine-grained token boundary selected beyond the last complete offload
    # chunk. It is consumed when the corresponding load is scheduled.
    partial_tail_boundary: int | None = None
    # True once a load from the offload tier was scheduled for this request.
    loaded_from_offload: bool = False
    # Loaded attention keys cover their token positions. Recurrent state
    # supports the whole reused external prefix.
    load_key_ranges: list[tuple[int, int, OffloadKey]] = field(default_factory=list)
    # True once on_request_finished has been signaled to the manager.
    finished_signaled: bool = False
    # A load for this request failed; all its later lookups, including after
    # preemption, skip the offloaded cache so a faulting load is not retried.
    load_failed: bool = False

    def __post_init__(self) -> None:
        self.group_states = tuple(
            RequestGroupState() for _ in self.config.kv_group_configs
        )
        params = self.req.kv_transfer_params

        # NOTE: This field is experimental and subject to change in the future.
        raw = params.get("max_load_tokens") if params else None
        if type(raw) is int and raw >= 0:
            self.max_load_tokens = raw
            logger.debug(
                "Request %s: max_load_tokens set to %d",
                self.req.request_id,
                raw,
            )
        elif raw is not None:
            logger.warning(
                "max_load_tokens must be a non-negative int, got %r; ignoring", raw
            )

        # NOTE: This field is experimental and subject to change in the future.
        raw = params.get("max_offload_tokens") if params else None
        if type(raw) is int and raw >= 0:
            self.max_offload_tokens = raw
            logger.debug(
                "Request %s: max_offload_tokens set to %d",
                self.req.request_id,
                raw,
            )
        elif raw is not None:
            logger.warning(
                "max_offload_tokens must be a non-negative int, got %r; ignoring", raw
            )

    def update_offload_keys(self) -> None:
        for group_config, group_state in zip(
            self.config.kv_group_configs, self.group_states
        ):
            stride = group_config.hashes_per_chunk
            start = stride * (len(group_state.offload_keys) + 1) - 1
            for req_block_hash in self.req.block_hashes[start::stride]:
                key = make_offload_key(req_block_hash, group_config.group_idx)
                group_state.offload_keys.append(key)
                self.req_context.set_offload_key_position(
                    key,
                    len(group_state.offload_keys) * group_config.tokens_per_chunk,
                )

    def update_block_id_groups(
        self, new_block_id_groups: tuple[list[int], ...] | None
    ) -> None:
        if new_block_id_groups is None:
            return

        for group_config, group_state in zip(
            self.config.kv_group_configs, self.group_states
        ):
            group_state.block_ids.extend(new_block_id_groups[group_config.group_idx])

    def storable_chunks(
        self,
        group_config: "GroupOffloadConfig",
        group_state: RequestGroupState,
        num_offloadable_tokens: int,
        apply_eagle_drop: bool,
    ) -> int:
        """Number of allocated and keyed leading chunks eligible for store.

        With legacy hashes, the volatile trailing chunk of an EAGLE/MTP group
        is excluded while decoding: the draft-layer KV of the last accepted
        position may be rewritten after spec-token rejection. During prefill
        the trailing chunk is stable (the draft input for a chunk's last
        position is the next prompt token), so it is stored immediately.
        Once the request has finished, no further spec-token rejection can
        rewrite the tail, so the exclusion is lifted and the final chunk
        becomes storable (issue #52735). The exclusion must be applied
        consistently everywhere the ON_COMPUTE progress is derived:
        otherwise the trailing chunk of each step is skipped on collection but
        jumped over by the progress, so it is never re-considered
        and a permanent hole breaks prefix-reuse lookup. `is_finished` is
        monotonic, so the finish-time calls all see the lifted exclusion.
        """
        num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
        is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
        # Finished requests have no pending speculation.
        if (
            apply_eagle_drop
            and group_config.is_eagle_group
            and is_decoding
            and not self.req.is_finished()
        ):
            num_chunks = max(0, num_chunks - 1)
        num_allocated_chunks = (
            len(group_state.block_ids) // self.config.blocks_per_chunk
        )
        num_keyed_chunks = len(group_state.offload_keys)
        return min(num_chunks, num_allocated_chunks, num_keyed_chunks)

    def num_offloadable_tokens(self, num_computed_tokens: int) -> int:
        num = min(num_computed_tokens, self.req.num_tokens)
        if self.config.num_hash_lookahead_tokens:
            num = min(
                num,
                len(self.req.block_hashes) * self.config.tokens_per_hash,
            )
        if self.max_offload_tokens is not None:
            num = min(num, self.max_offload_tokens)
        if self.config.offload_prompt_only:
            num = min(num, self.req.num_prompt_tokens)
        return num

    def register_boundary_key(self, group_idx: int, boundary_tokens: int) -> OffloadKey:
        hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
        key = make_offload_key(self.req.block_hashes[hash_idx], group_idx)
        self.req_context.set_offload_key_position(key, boundary_tokens)
        return key

    def update_num_hit_chunks(self, num_cached_tokens: int) -> None:
        for group_config, group_state in zip(
            self.config.kv_group_configs, self.group_states
        ):
            group_state.num_hit_chunks = (
                num_cached_tokens // group_config.tokens_per_chunk
            )

storable_chunks(group_config, group_state, num_offloadable_tokens, apply_eagle_drop)

Number of allocated and keyed leading chunks eligible for store.

With legacy hashes, the volatile trailing chunk of an EAGLE/MTP group is excluded while decoding: the draft-layer KV of the last accepted position may be rewritten after spec-token rejection. During prefill the trailing chunk is stable (the draft input for a chunk's last position is the next prompt token), so it is stored immediately. Once the request has finished, no further spec-token rejection can rewrite the tail, so the exclusion is lifted and the final chunk becomes storable (issue #52735). The exclusion must be applied consistently everywhere the ON_COMPUTE progress is derived: otherwise the trailing chunk of each step is skipped on collection but jumped over by the progress, so it is never re-considered and a permanent hole breaks prefix-reuse lookup. is_finished is monotonic, so the finish-time calls all see the lifted exclusion.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/state.py
def storable_chunks(
    self,
    group_config: "GroupOffloadConfig",
    group_state: RequestGroupState,
    num_offloadable_tokens: int,
    apply_eagle_drop: bool,
) -> int:
    """Number of allocated and keyed leading chunks eligible for store.

    With legacy hashes, the volatile trailing chunk of an EAGLE/MTP group
    is excluded while decoding: the draft-layer KV of the last accepted
    position may be rewritten after spec-token rejection. During prefill
    the trailing chunk is stable (the draft input for a chunk's last
    position is the next prompt token), so it is stored immediately.
    Once the request has finished, no further spec-token rejection can
    rewrite the tail, so the exclusion is lifted and the final chunk
    becomes storable (issue #52735). The exclusion must be applied
    consistently everywhere the ON_COMPUTE progress is derived:
    otherwise the trailing chunk of each step is skipped on collection but
    jumped over by the progress, so it is never re-considered
    and a permanent hole breaks prefix-reuse lookup. `is_finished` is
    monotonic, so the finish-time calls all see the lifted exclusion.
    """
    num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
    is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
    # Finished requests have no pending speculation.
    if (
        apply_eagle_drop
        and group_config.is_eagle_group
        and is_decoding
        and not self.req.is_finished()
    ):
        num_chunks = max(0, num_chunks - 1)
    num_allocated_chunks = (
        len(group_state.block_ids) // self.config.blocks_per_chunk
    )
    num_keyed_chunks = len(group_state.offload_keys)
    return min(num_chunks, num_allocated_chunks, num_keyed_chunks)

SchedulerOffloadConfig

Bases: NamedTuple

Methods:

Attributes:

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/state.py
class SchedulerOffloadConfig(NamedTuple):
    kv_group_configs: tuple[GroupOffloadConfig, ...]
    blocks_per_chunk: int
    tokens_per_hash: int
    num_workers: int
    offload_prompt_only: bool
    supports_partial_tail: bool
    alignment_tokens: int | None = None
    retention_interval: int | None = None
    dcp_world_size: int = 1
    sync_load_budget_bytes: int = 0
    # KV bytes per token on one worker, summed over the offloaded groups.
    kv_bytes_per_token: int = 0

    # Per group, the tokens past a partial-tail boundary whose hash keys the
    # group's partial-tail block. Non-zero only for EAGLE groups.
    eagle_proof_margins: tuple[int, ...] = ()
    # Lookahead block hashes already cover positions past the computed tokens.
    num_hash_lookahead_tokens: int = 0
    cache_hit_alignment_tokens: int | None = None
    drop_eagle_checkpoint_block: bool = False

    @property
    def supports_handoff(self) -> bool:
        """Whether the KV cache manager can hand off state blocks."""
        return any(cfg.requires_cow_source for cfg in self.kv_group_configs)

    @property
    def partial_tail_block_size(self) -> int:
        if not self.supports_partial_tail:
            return 0
        assert self.alignment_tokens is not None
        return self.alignment_tokens

    @property
    def max_eagle_proof_margin(self) -> int:
        return max(self.eagle_proof_margins, default=0)

    def partial_tail_key_tokens(self, config_group_idx: int, boundary: int) -> int:
        """Token position whose hash keys a group's partial-tail block."""
        if not self.eagle_proof_margins:
            return boundary
        return boundary + self.eagle_proof_margins[config_group_idx]

    @classmethod
    def from_spec(
        cls,
        spec: OffloadingSpec,
        vllm_config: VllmConfig,
        kv_cache_config: KVCacheConfig,
    ) -> "SchedulerOffloadConfig":
        # Determine the alignment token count from the full-attention group(s).
        # This is the tokens_per_chunk of the full-attention group; load
        # hits are always aligned to this boundary, so SWA blocks earlier in
        # each segment can never serve a load hit. Relevant for hybrid
        # architectures like DeepSeek V4 (MLA + SWA groups).
        full_attn_tokens_per_chunk: set[int] = set()
        for group in spec.config.groups:
            if group.sliding_window_size_in_chunks is None:
                full_attn_tokens_per_chunk.add(
                    group.tokens_per_block * spec.blocks_per_chunk
                )

        # Only apply the optimization if there's a single consistent
        # full-attention alignment size.  For pure SWA/Mamba models (no
        # full-attention groups), fall back to the largest SWA/Mamba
        # tokens_per_chunk so retention_interval has a boundary
        # granularity to work with.
        alignment_tokens: int | None = None
        if len(full_attn_tokens_per_chunk) == 1:
            alignment_tokens = full_attn_tokens_per_chunk.pop()
        elif not full_attn_tokens_per_chunk:
            all_tokens_per_chunk = [
                tpb * spec.blocks_per_chunk for tpb in spec.tokens_per_block
            ]
            if all_tokens_per_chunk:
                alignment_tokens = max(all_tokens_per_chunk)

        retention_interval = vllm_config.cache_config.prefix_cache_retention_interval

        eagle_groups = {
            group.group_id
            for group in spec.config.groups
            if kv_cache_config.kv_cache_groups[group.group_id].is_eagle_group
        }

        use_eagle_block_drop = (
            vllm_config.speculative_config is not None
            and vllm_config.speculative_config.use_eagle_block_drop()
        )
        if use_eagle_block_drop and not eagle_groups:
            # No group is annotated as holding drafter layers. This happens
            # when the drafter owns no KV cache (e.g. KV-sharing drafters) or
            # the model runner does not tag draft layers' KV cache specs.
            # Treating every group as a drafter group here makes
            # the volatile-tail pop apply to ALL groups, which can zero the
            # whole request's offload hit whenever any group's servable
            # window is a single chunk (issue #52735). Drafter KV served one
            # chunk stale can only lower speculative acceptance rates — the
            # target model verifies every draft — so fail toward serving.
            logger.info_once(
                "KV offloading: speculative decoding is enabled but no "
                "KV-cache group is annotated as a drafter group; treating "
                "all groups as non-draft for offloading."
            )

        if eagle_groups:
            logger.info(
                "KV offloading: EAGLE/MTP draft attention groups %s detected.",
                sorted(eagle_groups),
            )

        kv_group_configs_list: list[GroupOffloadConfig] = []
        for group in spec.config.groups:
            group_id = group.group_id
            tokens_per_block = group.tokens_per_block
            kv_cache_group = kv_cache_config.kv_cache_groups[group_id]
            kv_group_configs_list.append(
                GroupOffloadConfig(
                    group_idx=group_id,
                    tokens_per_block=tokens_per_block,
                    tokens_per_chunk=tokens_per_block * spec.blocks_per_chunk,
                    hashes_per_chunk=(
                        (tokens_per_block * spec.blocks_per_chunk)
                        // spec.tokens_per_hash
                    ),
                    sliding_window_size_in_chunks=group.sliding_window_size_in_chunks,
                    block_size=kv_cache_group.block_size,
                    manager_config=kv_cache_group.manager_config,
                    kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
                    is_eagle_group=group_id in eagle_groups,
                    requires_cow_source=is_mamba_align_group(kv_cache_group),
                )
            )
        kv_group_configs = tuple(kv_group_configs_list)
        has_partial_recurrent_group = any(
            config.requires_cow_source for config in kv_group_configs
        )
        # Store sub-block tails only when core can reuse them. Recurrent blocks
        # stay unsharded under DCP and must divide the full-attention chunk.
        # EAGLE proof positions are not validated under DCP yet.
        dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
        cache_hit_alignment = kv_cache_config.cache_hit_alignment_tokens
        supports_partial_tail = (
            spec.blocks_per_chunk == 1
            and alignment_tokens is not None
            and cache_hit_alignment is not None
            and cache_hit_alignment < alignment_tokens
            and has_partial_recurrent_group
            and all(
                config.sliding_window_size_in_chunks is None
                or (
                    config.requires_cow_source
                    and alignment_tokens % config.tokens_per_block == 0
                    and config.tokens_per_block % spec.tokens_per_hash == 0
                )
                for config in kv_group_configs
            )
            and spec.tokens_per_hash % dcp_world_size == 0
            and (
                dcp_world_size == 1
                or not any(config.is_eagle_group for config in kv_group_configs)
            )
        )

        # Core's prefix cache matches an EAGLE group this many tokens past a
        # hit and then drops them. A partial tail stores and looks up EAGLE
        # groups at that proof position, with the margin rule of the core
        # coordinator. Lookahead block hashes already key that position, so
        # the groups need no margin.
        eagle_proof_margins: tuple[int, ...] = ()
        if supports_partial_tail and vllm_config.num_hash_lookahead_tokens == 0:
            assert alignment_tokens is not None
            partial_hash_hits = (
                cache_hit_alignment is not None
                and cache_hit_alignment < alignment_tokens
            )
            eagle_proof_margins = tuple(
                eagle_proof_margin(
                    alignment_tokens,
                    spec.tokens_per_hash,
                    partial_hash_hits
                    and config.manager_cls.supports_fine_grained_hash_lookup,
                )
                if config.is_eagle_group and not config.requires_cow_source
                else 0
                for config in kv_group_configs
            )

        if retention_interval is not None:
            if retention_interval < 0:
                raise ValueError(
                    f"prefix_cache_retention_interval "
                    f"({retention_interval}) must be non-negative."
                )
            for config in kv_group_configs:
                if (
                    config.sliding_window_size_in_chunks is not None
                    and retention_interval % config.tokens_per_chunk != 0
                ):
                    raise ValueError(
                        f"prefix_cache_retention_interval "
                        f"({retention_interval}) must be a multiple of "
                        f"tokens_per_chunk ({config.tokens_per_chunk})."
                    )

        return cls(
            num_workers=vllm_config.parallel_config.world_size,
            kv_group_configs=kv_group_configs,
            blocks_per_chunk=spec.blocks_per_chunk,
            tokens_per_hash=spec.tokens_per_hash,
            offload_prompt_only=spec.offload_prompt_only,
            supports_partial_tail=supports_partial_tail,
            alignment_tokens=alignment_tokens,
            retention_interval=retention_interval,
            dcp_world_size=dcp_world_size,
            # Per-step KV bytes (per worker) loaded within the step (0 disables).
            sync_load_budget_bytes=int(
                spec.extra_config.get("sync_load_budget_bytes", 0)
            ),
            kv_bytes_per_token=sum(
                kv_cache_config.kv_cache_groups[g.group_id].page_size_bytes
                // g.tokens_per_block
                for g in spec.config.groups
            ),
            eagle_proof_margins=eagle_proof_margins,
            num_hash_lookahead_tokens=vllm_config.num_hash_lookahead_tokens,
            cache_hit_alignment_tokens=cache_hit_alignment,
            drop_eagle_checkpoint_block=(
                use_eagle_block_drop and vllm_config.num_hash_lookahead_tokens == 0
            ),
        )

supports_handoff property

Whether the KV cache manager can hand off state blocks.

partial_tail_key_tokens(config_group_idx, boundary)

Token position whose hash keys a group's partial-tail block.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/state.py
def partial_tail_key_tokens(self, config_group_idx: int, boundary: int) -> int:
    """Token position whose hash keys a group's partial-tail block."""
    if not self.eagle_proof_margins:
        return boundary
    return boundary + self.eagle_proof_margins[config_group_idx]

resolve_mamba_align_size(spec, kv_cache_config)

Scan all KV cache groups in spec and return the single mamba alignment size, or None if no group requires mamba alignment.

For Mamba groups in "align" cache mode the hit window must be rounded down to a multiple of the offloaded chunk size. Asserts that all such groups agree on the same value.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/state.py
def resolve_mamba_align_size(
    spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
) -> int | None:
    """Scan all KV cache groups in *spec* and return the single mamba alignment
    size, or None if no group requires mamba alignment.

    For Mamba groups in "align" cache mode the hit window must be
    rounded down to a multiple of the offloaded chunk size. Asserts that all
    such groups agree on the same value.
    """
    mamba_align_size: int | None = None
    for group in spec.config.groups:
        tokens_per_block = group.tokens_per_block
        if is_mamba_align_group(kv_cache_config.kv_cache_groups[group.group_id]):
            tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
            assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
            mamba_align_size = tokens_per_chunk
    return mamba_align_size