Skip to content

vllm.models.deepseek_v41.nvidia.pipeline

Pipeline dependency planning, cache replicas and intermediate tensor transport.

Classes:

  • PipelineCache –

    A source-named replica participates in the same scheduler cache group.

Functions:

PipelineCache

Bases: Module, AttentionLayerBase

A source-named replica participates in the same scheduler cache group.

Source code in vllm/models/deepseek_v41/nvidia/pipeline.py
class PipelineCache(nn.Module, AttentionLayerBase):
    """A source-named replica participates in the same scheduler cache group."""

    def __init__(self, vllm_config: VllmConfig, prefix: str):
        super().__init__()
        self.kv_cache = torch.tensor([])
        self.spec: KVCacheSpec | None = None
        self.backend: type[AttentionBackend] | None = None
        context = vllm_config.compilation_config.static_forward_context
        assert prefix not in context
        context[prefix] = self

    def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
        self.kv_cache = kv_cache.squeeze(1)

    def get_kv_cache_config(self, vllm_config: VllmConfig) -> KVCacheLayerConfig:
        assert self.spec is not None
        # The executor may update the block size after model construction.
        spec = replace(self.spec, block_size=vllm_config.cache_config.block_size)
        return FullAttentionManager.layer_config(spec)

    def get_attn_backend(self) -> type[AttentionBackend]:
        assert self.backend is not None
        return self.backend

    def forward(self):
        raise RuntimeError("Pipeline cache replicas do not execute attention")

get_pipeline_boundaries(config, stage_ranges)

Resolve cache, top-k and candidate lifetimes at every stage cut.

Source code in vllm/models/deepseek_v41/nvidia/pipeline.py
def get_pipeline_boundaries(
    config: Any, stage_ranges: list[tuple[int, int]]
) -> tuple[PipelineBoundary, ...]:
    """Resolve cache, top-k and candidate lifetimes at every stage cut."""
    num_layers = config.num_hidden_layers
    if (
        not stage_ranges
        or stage_ranges[0][0] != 0
        or stage_ranges[-1][1] != num_layers
        or any(start >= end for start, end in stage_ranges)
        or any(a[1] != b[0] for a, b in zip(stage_ranges, stage_ranges[1:]))
    ):
        raise ValueError("DeepSeek-V4.1 pipeline stages must partition all layers")
    ratios = config.compress_ratios
    if len(ratios) < num_layers or any(r not in (0, 1, 2) for r in ratios):
        raise ValueError("Invalid DeepSeek-V4.1 compress_ratios")
    sources = {}
    for kind, field in (
        ("kv", "kv_source_layer_ids"),
        ("index", "index_source_layer_ids"),
    ):
        values = tuple(getattr(config, field, ()) or ())
        if (
            values != tuple(sorted(set(values)))
            or any(type(s) is not int or not 0 <= s < num_layers for s in values)
            or any(ratios[s] == 0 for s in values)
        ):
            raise ValueError(f"Invalid DeepSeek-V4.1 {field}")
        sources[kind] = values
    if not set(sources["kv"]).issubset(sources["index"]):
        raise ValueError("KV sources must also publish index keys")
    dependencies: dict[str, list[tuple[int, int]]] = {"kv": [], "index": []}
    for layer in range(num_layers):
        if ratios[layer] == 0:
            continue
        for kind, values in sources.items():
            preceding = [s for s in values if s <= layer]
            if not preceding:
                raise ValueError(f"Layer {layer} has no preceding {kind} source")
            source = preceding[-1]
            if ratios[source] != ratios[layer]:
                raise ValueError(
                    f"Layer {layer} and its {kind} source disagree on ratio"
                )
            dependencies[kind].append((source, layer))
    kv_source_of = {layer: source for source, layer in dependencies["kv"]}
    dependencies["indexer_k"] = [
        (kv_source_of[layer], layer)
        for layer in sources["index"]
        if layer not in sources["kv"]
    ]
    candidate = getattr(config, "candidate_source_layer_id", -1)
    candidate_enabled = (
        candidate >= 0 and getattr(config, "candidate_topk_blocks", 0) > 0
    )
    if candidate_enabled and candidate not in sources["index"]:
        raise ValueError("Candidate source must be an index source")
    boundaries = []
    for _, cut in stage_ranges[:-1]:
        live = {}
        for kind, edges in dependencies.items():
            crossing = {s for s, consumer in edges if s < cut <= consumer}
            assert len(crossing) <= 1
            live[kind] = next(iter(crossing), None)
        boundaries.append(
            PipelineBoundary(
                kv_source=live["kv"],
                index_source=live["index"],
                candidate_source=(
                    candidate
                    if candidate_enabled
                    and candidate < cut
                    and any(s >= cut for s in sources["index"])
                    else None
                ),
                indexer_k_source=live["indexer_k"],
            )
        )
    return tuple(boundaries)