Skip to content

vllm.distributed.kv_transfer.kv_connector.v1.offloading.common

Classes:

OffloadingWorkerMetadata dataclass

Bases: KVConnectorWorkerMetadata

Worker -> Scheduler metadata for completed transfer jobs.

Each worker reports {job_id: 1} for newly completed transfer jobs (load or store). aggregate() sums counts across workers within a step. The scheduler accumulates across steps and processes a transfer completion only when count reaches num_workers.

Methods:

  • mark_completed –

    Record a transfer job completion from this worker.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py
@dataclass
class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
    """Worker -> Scheduler metadata for completed transfer jobs.

    Each worker reports {job_id: 1} for newly completed transfer jobs
    (load or store). aggregate() sums counts across workers within a step.
    The scheduler accumulates across steps and processes
    a transfer completion only when count reaches num_workers.
    """

    completed_jobs: dict[int, int] = field(default_factory=dict)
    failed_jobs: set[int] = field(default_factory=set)
    transfer_stats: TransferStats = field(default_factory=TransferStats)
    # Ranks of workers that became ready, under async_init. Each reports once.
    ready_ranks: set[int] = field(default_factory=set)

    def mark_completed(self, job_id: int, success: bool = True) -> None:
        """Record a transfer job completion from this worker."""
        self.completed_jobs[job_id] = 1
        if not success:
            self.failed_jobs.add(job_id)

    def aggregate(
        self, other: "KVConnectorWorkerMetadata"
    ) -> "KVConnectorWorkerMetadata":
        assert isinstance(other, OffloadingWorkerMetadata)

        merged = dict(self.completed_jobs)
        for job_id, v in other.completed_jobs.items():
            merged[job_id] = merged.get(job_id, 0) + v

        return OffloadingWorkerMetadata(
            completed_jobs=merged,
            failed_jobs=self.failed_jobs | other.failed_jobs,
            transfer_stats=self.transfer_stats.aggregate(other.transfer_stats),
            ready_ranks=self.ready_ranks | other.ready_ranks,
        )

mark_completed(job_id, success=True)

Record a transfer job completion from this worker.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py
def mark_completed(self, job_id: int, success: bool = True) -> None:
    """Record a transfer job completion from this worker."""
    self.completed_jobs[job_id] = 1
    if not success:
        self.failed_jobs.add(job_id)

TransferJob dataclass

A transfer job bundling request context with transfer spec.

Used for both loads and stores, keyed by scheduler-assigned job ID. The worker reports the job ID back when the transfer finishes, and the scheduler processes the completion.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py
@dataclass(slots=True)
class TransferJob:
    """A transfer job bundling request context with transfer spec.

    Used for both loads and stores, keyed by scheduler-assigned job ID.
    The worker reports the job ID back when the transfer finishes,
    and the scheduler processes the completion.
    """

    req_id: ReqId
    src_spec: LoadStoreSpec
    dst_spec: LoadStoreSpec
    # Load only: completed before this step's forward pass.
    sync: bool = False

TransferJobStatus dataclass

Tracks scheduler-side state for a single transfer job.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py
@dataclass(slots=True)
class TransferJobStatus:
    """Tracks scheduler-side state for a single transfer job."""

    req_id: ReqId
    # Number of workers still pending. Starts at num_workers,
    # decremented as each worker reports completion. Job is done at 0.
    pending_count: int
    # Offload keys this job covers, in prefix order; passed to
    # manager.complete_*().
    keys: tuple[OffloadKey, ...]
    is_store: bool
    # Unlike an async load, store jobs may be created next to it.
    is_sync_load: bool = False
    failed: bool = False
    # Store source blocks fenced after the request finishes.
    deferred_fence_block_ids: list[int] | None = None
    # Store source blocks fenced when the transfer is created.
    fenced_block_ids: list[int] | None = None
    # The trigger of the handler that made a store job. The scheduler sets it.
    trigger: StoreTrigger | None = None