Skip to content

vllm.multimodal.video_decoders

Modules:

Classes:

Functions:

SampledFramesCharge

Charges the decoded size of sampled video frames to a request budget.

VideoMediaIO installs one with charging_sampled_frames while a loader runs. Decoders call charge_sampled_frames once they know the frame size and the sampled indices, so an oversized sample is rejected before any frame is decoded.

Source code in vllm/multimodal/video_decoders/base.py
class SampledFramesCharge:
    """Charges the decoded size of sampled video frames to a request budget.

    `VideoMediaIO` installs one with `charging_sampled_frames` while a loader
    runs. Decoders call `charge_sampled_frames` once they know the frame
    size and the sampled indices, so an oversized sample is rejected before
    any frame is decoded.
    """

    def __init__(self, charge: Callable[[int], None]) -> None:
        self._charge = charge
        self.charged_bytes = 0

    def __call__(self, num_bytes: int) -> None:
        self._charge(num_bytes)
        self.charged_bytes += num_bytes

VideoSourceMetadata

Bases: NamedTuple

Metadata describing the encoded video source.

Source code in vllm/multimodal/video_decoders/base.py
class VideoSourceMetadata(NamedTuple):
    """Metadata describing the encoded video source."""

    total_frames_num: int
    original_fps: float
    duration: float

VideoTargetMetadata

Bases: NamedTuple

Metadata describing the requested video sample.

Source code in vllm/multimodal/video_decoders/base.py
class VideoTargetMetadata(NamedTuple):
    """Metadata describing the requested video sample."""

    num_frames: int
    fps: float
    max_duration: float

charge_frame_copies(num_copies, frame_bytes)

Charge num_copies copies of a decoded frame of frame_bytes bytes.

Loaders that repeat decoded frames call this before allocating the copies, so the copies count toward the per-request decoded-media budget before they exist.

Source code in vllm/multimodal/video_decoders/base.py
def charge_frame_copies(num_copies: int, frame_bytes: int) -> None:
    """Charge `num_copies` copies of a decoded frame of `frame_bytes` bytes.

    Loaders that repeat decoded frames call this before allocating the
    copies, so the copies count toward the per-request decoded-media budget
    before they exist.
    """
    charge = _sampled_frames_charge.get()
    if charge is not None:
        charge(max(0, num_copies) * max(0, frame_bytes))

charge_sampled_frames(num_frames, width, height)

Charge num_frames RGB frames of width x height before decoding.

Raises ValueError when the frames would exceed the per-request decoded-media budget (VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES).

Source code in vllm/multimodal/video_decoders/base.py
def charge_sampled_frames(num_frames: int, width: int, height: int) -> None:
    """Charge `num_frames` RGB frames of `width` x `height` before decoding.

    Raises `ValueError` when the frames would exceed the per-request
    decoded-media budget (`VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES`).
    """
    charge = _sampled_frames_charge.get()
    if charge is not None:
        charge(max(0, num_frames) * max(0, width) * max(0, height) * 3)

charging_sampled_frames(charge)

Route charge_sampled_frames calls made in this context to charge.

Source code in vllm/multimodal/video_decoders/base.py
@contextmanager
def charging_sampled_frames(charge: SampledFramesCharge | None) -> Iterator[None]:
    """Route `charge_sampled_frames` calls made in this context to `charge`."""
    token = _sampled_frames_charge.set(charge)
    try:
        yield
    finally:
        _sampled_frames_charge.reset(token)

check_frame_pixel_limit(width, height)

Reject video frames exceeding VLLM_MAX_IMAGE_PIXELS.

Source code in vllm/multimodal/video_decoders/base.py
def check_frame_pixel_limit(width: int, height: int) -> None:
    """Reject video frames exceeding `VLLM_MAX_IMAGE_PIXELS`."""
    max_pixels = envs.VLLM_MAX_IMAGE_PIXELS
    if max_pixels > 0 and width * height > max_pixels:
        raise ValueError(
            f"Video frame dimensions {width}x{height} "
            f"({width * height} pixels) exceed the maximum of "
            f"{max_pixels} pixels. Set VLLM_MAX_IMAGE_PIXELS to "
            f"increase this limit."
        )

decode_video(backend, loader_cls, data, target, sampling_kwargs, backend_kwargs, *, frame_recovery)

Decode a sampled video, importing only the selected backend.

Source code in vllm/multimodal/video_decoders/__init__.py
def decode_video(
    backend: str,
    loader_cls,
    data: bytes,
    target: VideoTargetMetadata,
    sampling_kwargs: dict[str, Any],
    backend_kwargs: dict[str, Any],
    *,
    frame_recovery: bool,
):
    """Decode a sampled video, importing only the selected backend."""
    _get_backend_option_defaults(backend)
    if frame_recovery and backend != "opencv":
        error = (
            ValueError if backend == PYNVVIDEOCODEC_VIDEO_BACKEND else AssertionError
        )
        raise error(f"frame_recovery is not supported by the {backend!r} backend")

    decoder_kwargs = dict(backend_kwargs)
    if backend == "opencv":
        decoder_kwargs["frame_recovery"] = frame_recovery
    module = import_module(f".{backend}", __name__)
    decoder = getattr(module, f"decode_{backend}")
    return decoder(
        loader_cls,
        data,
        target,
        sampling_kwargs,
        **decoder_kwargs,
    )

incompatible_backend_options(old_backend, new_backend)

Option names accepted by old_backend but not by new_backend.

Source code in vllm/multimodal/video_decoders/__init__.py
def incompatible_backend_options(
    old_backend: str | None,
    new_backend: str,
) -> set[str]:
    """Option names accepted by `old_backend` but not by `new_backend`."""
    old = _BACKEND_OPTION_DEFAULTS.get(old_backend or "", {})
    new = _BACKEND_OPTION_DEFAULTS.get(new_backend, {})
    return old.keys() - new.keys()

resolve_video_backend_kwargs(backend, kwargs)

Split frame-sampling kwargs from options owned by a decoder backend.

Source code in vllm/multimodal/video_decoders/__init__.py
def resolve_video_backend_kwargs(
    backend: str,
    kwargs: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, Any]]:
    """Split frame-sampling kwargs from options owned by a decoder backend."""
    defaults = _get_backend_option_defaults(backend)
    sampling_kwargs = dict(kwargs)
    backend_kwargs = dict(defaults)
    backend_option_names = {
        name for options in _BACKEND_OPTION_DEFAULTS.values() for name in options
    }
    misplaced = (sampling_kwargs.keys() & backend_option_names) - defaults.keys()
    if misplaced:
        names = ", ".join(sorted(misplaced))
        raise ValueError(f"{names} is not supported by the {backend!r} backend")

    for name in defaults:
        if name in sampling_kwargs:
            backend_kwargs[name] = sampling_kwargs.pop(name)

    return sampling_kwargs, backend_kwargs