Skip to content

vllm.model_executor.model_loader.weight_cache.weight_seed.mooncake

The Mooncake weight mover: pull storages through the TransferEngine.

Classes:

  • MooncakeMover –

    Pull storages through the Mooncake TransferEngine.

MooncakeMover

Bases: WeightMover

Pull storages through the Mooncake TransferEngine.

Source code in vllm/model_executor/model_loader/weight_cache/weight_seed/mooncake.py
class MooncakeMover(WeightMover):
    """Pull storages through the Mooncake TransferEngine."""

    name = MOONCAKE_BACKEND

    def __init__(self, protocol: str = "rdma"):
        self.protocol = protocol
        self._engine = None
        self._source_session: str | None = None
        # Mooncake refuses to register memory twice, so each storage is
        # registered once and serves every later mirror.
        self._registered: dict[int, int] = {}

    def _initialize_engine(self):
        engine = TransferEngine()
        if engine.initialize(get_ip(), "P2PHANDSHAKE", self.protocol, "") != 0:
            raise RuntimeError("Mooncake TransferEngine initialization failed")
        return engine, f"{get_ip()}:{engine.get_rpc_port()}"

    def _get_engine(self):
        if self._engine is None:
            self._engine, self._source_session = self._initialize_engine()
        return self._engine

    def prepare_seed(self, storages: list[torch.UntypedStorage]) -> dict:
        engine = self._get_engine()
        regions = [[storage.data_ptr(), storage.nbytes()] for storage in storages]
        new = [
            (pointer, nbytes)
            for pointer, nbytes in regions
            if nbytes and pointer not in self._registered
        ]
        if new:
            pointers, lengths = (list(column) for column in zip(*new))
            if engine.batch_register_memory(pointers, lengths) != 0:
                raise RuntimeError("Mooncake failed to register weight-cache storages")
            self._registered.update(new)
        return {
            "backend": self.name,
            "session": self._source_session,
            "regions": regions,
        }

    def copy_into(
        self,
        storages: list[Mapping],
        seed: Mapping,
        buffers: list[torch.Tensor],
    ) -> None:
        session = seed.get("session")
        if not session:
            raise RuntimeError("Source daemon did not publish a Mooncake session")
        regions = seed.get("regions")
        if not isinstance(regions, list) or len(regions) != len(storages):
            raise RuntimeError("Source daemon published no region per storage")
        transfers = [
            (buffer.data_ptr(), remote_pointer, buffer.nbytes)
            for buffer, (remote_pointer, _) in zip(buffers, regions)
            if buffer.nbytes
        ]
        if not transfers:
            return
        local_pointers, remote_pointers, lengths = (
            list(column) for column in zip(*transfers)
        )
        engine, _ = self._initialize_engine()
        if engine.batch_register_memory(local_pointers, lengths) != 0:
            raise RuntimeError("Mooncake failed to register mirror storages")

        try:
            ret = engine.batch_transfer_sync_read(
                session, local_pointers, remote_pointers, lengths
            )
            if ret != 0:
                raise RuntimeError(f"Mooncake weight-cache read failed: {ret}")
            ret = engine.batch_unregister_memory(local_pointers)
            if ret != 0:
                raise RuntimeError(f"Mooncake mirror unregistration failed: {ret}")
        except Exception:
            # A failed call may leave tasks or registrations using the buffers.
            _retained_copies.append((engine, buffers))
            raise

    def release(self) -> None:
        if self._engine is not None and self._registered:
            ret = self._engine.batch_unregister_memory(list(self._registered))
            if ret != 0:
                raise RuntimeError(
                    "Mooncake failed to unregister weight-cache storages with "
                    f"status {ret}; registrations were retained"
                )
            self._registered.clear()