Skip to content

vllm.model_executor.models.lfm3_omni

Inference-only LFM3-Omni model: images, videos and speech in, text out.

Prompts follow the Transformers processor: every image, video and clip is wrapped in one span pair, a video is a series of timestamped frame groups, and with use_audio_in_video each video takes the first clip after it as its soundtrack, split across its frame groups. The speech output head is not served and its weights are skipped on load.

Classes:

Lfm3OmniMultiModalProcessor

Bases: Qwen3VLMultiModalProcessor

Source code in vllm/model_executor/models/lfm3_omni.py
class Lfm3OmniMultiModalProcessor(Qwen3VLMultiModalProcessor):
    info: Lfm3OmniProcessingInfo

    def _get_mm_fields_config(
        self, hf_inputs: BatchFeature, hf_processor_mm_kwargs: Mapping[str, object]
    ) -> Mapping[str, MultiModalFieldConfig]:
        return {
            **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
            "input_features": MultiModalFieldConfig.batched("audio"),
            "audio_embeds": MultiModalFieldConfig.batched("audio"),
        }

    def _postprocess_hf_mm_data(
        self,
        mm_inputs: Mapping[str, object],
        hf_processor_mm_kwargs: Mapping[str, object],
        processed_data: BatchFeature,
    ) -> BatchFeature:
        return split_audio_features(processed_data)

    def _span_token_ids(self) -> tuple[int, int]:
        processor = self.info.get_hf_processor()
        return processor.vision_start_token_id, processor.vision_end_token_id

    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
        # Each placeholder gets exactly one span pair, so drop the pair a chat
        # template already put around it.
        processor = self.info.get_hf_processor()
        start, end = self._span_token_ids()
        placeholders = {
            processor.image_token_id,
            processor.video_token_id,
            processor.audio_token_id,
        }
        stripped: list[int] = []
        index = 0
        while index < len(prompt):
            if (
                prompt[index] == start
                and index + 2 < len(prompt)
                and prompt[index + 1] in placeholders
                and prompt[index + 2] == end
            ):
                stripped.append(prompt[index + 1])
                index += 3
            else:
                stripped.append(prompt[index])
                index += 1
        return stripped

    def _wrap(self, tokens: list[int]) -> list[int]:
        start, end = self._span_token_ids()
        return [start, *tokens, end]

    def _with_soundtrack(
        self, video: _VideoGroups, num_tokens: int
    ) -> PromptUpdateDetails:
        """Splits the soundtrack's encoder tokens at the first frame of each
        group, after the group."""
        processor = self.info.get_hf_processor()
        extractor = processor.feature_extractor
        seconds_per_token = (
            extractor.hop_length * extractor.encoder_stride / extractor.sampling_rate
        )
        bounds = [0] + [
            min(num_tokens, round(start / seconds_per_token))
            for start in video.group_starts[1:]
        ]
        bounds = [max(bounds[: i + 1]) for i in range(len(bounds))] + [num_tokens]
        tokens: list[int] = []
        for group, first, last in zip(video.groups, bounds, bounds[1:]):
            tokens += group + [processor.audio_token_id] * (last - first)
        return PromptUpdateDetails.select_token_id(
            self._wrap(tokens), processor.video_token_id
        )

    def _get_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> Sequence[PromptUpdate]:
        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
        tokenizer = self.info.get_tokenizer()
        extractor = processor.feature_extractor
        start, end = self._span_token_ids()
        temporal_patch_size = processor.video_processor.temporal_patch_size
        merge_length = processor.image_processor.merge_size**2
        use_audio_in_video = bool(hf_processor_mm_kwargs.get("use_audio_in_video"))

        def wrapped(token_id: int, count: int) -> PromptUpdateDetails:
            return PromptUpdateDetails.select_token_id(
                self._wrap([token_id] * count), token_id
            )

        def image_replacement(item_idx: int) -> PromptUpdateDetails:
            grid_thw = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
            assert isinstance(grid_thw, torch.Tensor)
            return wrapped(
                processor.image_token_id, int(grid_thw.prod()) // merge_length
            )

        def audio_replacement(item_idx: int) -> PromptUpdateDetails:
            item = out_mm_kwargs["audio"][item_idx]
            if (embeds := item.get("audio_embeds")) is not None:
                count = len(embeds.data)
            else:
                features = item["input_features"].data
                assert isinstance(features, torch.Tensor)
                count = extractor.num_encoder_tokens(features.shape[-1])
            return wrapped(processor.audio_token_id, count)

        def video_replacement(item_idx: int) -> PromptUpdateDetails:
            item = out_mm_kwargs["video"][item_idx]
            grid_thw = item["video_grid_thw"].data
            assert isinstance(grid_thw, torch.Tensor)
            timestamps = item["timestamps"].data
            assert isinstance(timestamps, (list, torch.Tensor))
            tokens_per_group = int(grid_thw[1:].prod()) // merge_length
            groups = [
                self.get_video_repl(
                    tokens_per_frame=[tokens_per_group],
                    timestamps=[timestamp],
                    tokenizer=tokenizer,
                    vision_start_token_id=start,
                    vision_end_token_id=end,
                    video_token_id=processor.video_token_id,
                ).full
                for timestamp in timestamps
            ]
            details = PromptUpdateDetails.select_token_id(
                self._wrap([token for group in groups for token in group]),
                processor.video_token_id,
            )
            if not use_audio_in_video:
                return details
            # The soundtrack splits at the first frame of each group, which the
            # averaged timestamps lose, so take the frame indices the loader
            # sampled; frames resampled by the HF processor change the count.
            videos = mm_items.get("video")
            video = (
                videos.get(item_idx)
                if isinstance(videos, VideoProcessorItems)
                else None
            )
            metadata = video[1] if isinstance(video, tuple) else {}
            indices = metadata.get("frames_indices", [])
            if -(-len(indices) // temporal_patch_size) != len(groups):
                raise ValueError(
                    "`use_audio_in_video` needs videos sampled by the video loader"
                )
            return _VideoGroups(
                full=details.full,
                is_embed=details.is_embed,
                groups=groups,
                group_starts=[
                    index / metadata["fps"] for index in indices[::temporal_patch_size]
                ],
            )

        return [
            PromptReplacement("image", [processor.image_token_id], image_replacement),
            PromptReplacement("video", [processor.video_token_id], video_replacement),
            PromptReplacement("audio", [processor.audio_token_id], audio_replacement),
        ]

    def _with_soundtrack_placeholders(
        self, prompt: list[int], num_audios: int
    ) -> list[int]:
        """The chat parser adds each video's decoded soundtrack as the audio
        item right after it; the openai content format puts no placeholder for
        it in the prompt. When the audio items outnumber the placeholders by
        exactly one per video, put that placeholder right after every video,
        as HF's `load_audio_from_video` does; otherwise the prompt is kept."""
        processor = self.info.get_hf_processor()
        video_token_id = processor.video_token_id
        audio_token_id = processor.audio_token_id
        num_videos = prompt.count(video_token_id)
        if num_audios != prompt.count(audio_token_id) + num_videos:
            return prompt
        with_tracks: list[int] = []
        for token in prompt:
            with_tracks.append(token)
            if token == video_token_id:
                with_tracks.append(audio_token_id)
        return with_tracks

    def _soundtracks(self, prompt: list[int]) -> dict[int, int]:
        """The audio item each video takes as its soundtrack: the first clip
        after the video that no earlier video took."""
        processor = self.info.get_hf_processor()
        clips = [i for i, t in enumerate(prompt) if t == processor.audio_token_id]
        taken: list[int] = []
        for video in (i for i, t in enumerate(prompt) if t == processor.video_token_id):
            clip_idx = next(
                (i for i, clip in enumerate(clips) if clip > video and i not in taken),
                None,
            )
            if clip_idx is None:
                raise ValueError(
                    "`use_audio_in_video` needs an audio clip after each video"
                )
            taken.append(clip_idx)
        return dict(enumerate(taken))

    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        mm_res: MultiModalProcessingResult,
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
        if not any(
            isinstance(update.content, _VideoGroups)
            for item in mm_res.prompt_updates.get("video", [])
            for update in item
        ):
            return super()._maybe_apply_prompt_updates(mm_items, mm_res)

        mm_item_counts = mm_items.get_all_counts()
        self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
        self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
        prompt = self._with_soundtrack_placeholders(
            mm_res.prompt_ids, mm_item_counts.get("audio", 0)
        )

        # A soundtrack's own placeholder expands to nothing; its tokens follow
        # its video's frame groups instead.
        soundtracks = self._soundtracks(prompt)
        updates: dict[str, list[list[ResolvedPromptUpdate]]] = {
            modality: [list(item) for item in items]
            for modality, items in mm_res.prompt_updates.items()
        }
        audio_token_id = self.info.get_hf_processor().audio_token_id
        for video_idx, audio_idx in soundtracks.items():
            (video,) = updates["video"][video_idx]
            (audio,) = updates["audio"][audio_idx]
            assert isinstance(video.content, _VideoGroups)
            content = self._with_soundtrack(
                video.content, audio.content.full.count(audio_token_id)
            )
            updates["video"][video_idx] = [replace(video, content=content)]
            updates["audio"][audio_idx] = [
                replace(audio, content=PromptUpdateDetails.from_seq([]))
            ]

        prompt_ids, placeholders = self._apply_prompt_updates(prompt, updates)
        audio_placeholders = list(placeholders.get("audio", []))
        for video_placeholder in placeholders["video"]:
            audio_idx = soundtracks[video_placeholder.item_idx]
            audio_placeholders.append(
                replace(
                    video_placeholder,
                    modality="audio",
                    item_idx=audio_idx,
                    is_embed=torch.tensor(video_placeholder.tokens) == audio_token_id,
                )
            )
        placeholders = {
            **placeholders,
            "audio": sorted(audio_placeholders, key=lambda info: info.item_idx),
        }
        self._validate_mm_placeholders(placeholders, mm_item_counts)
        return prompt_ids, placeholders

_soundtracks(prompt)

The audio item each video takes as its soundtrack: the first clip after the video that no earlier video took.

Source code in vllm/model_executor/models/lfm3_omni.py
def _soundtracks(self, prompt: list[int]) -> dict[int, int]:
    """The audio item each video takes as its soundtrack: the first clip
    after the video that no earlier video took."""
    processor = self.info.get_hf_processor()
    clips = [i for i, t in enumerate(prompt) if t == processor.audio_token_id]
    taken: list[int] = []
    for video in (i for i, t in enumerate(prompt) if t == processor.video_token_id):
        clip_idx = next(
            (i for i, clip in enumerate(clips) if clip > video and i not in taken),
            None,
        )
        if clip_idx is None:
            raise ValueError(
                "`use_audio_in_video` needs an audio clip after each video"
            )
        taken.append(clip_idx)
    return dict(enumerate(taken))

_with_soundtrack(video, num_tokens)

Splits the soundtrack's encoder tokens at the first frame of each group, after the group.

Source code in vllm/model_executor/models/lfm3_omni.py
def _with_soundtrack(
    self, video: _VideoGroups, num_tokens: int
) -> PromptUpdateDetails:
    """Splits the soundtrack's encoder tokens at the first frame of each
    group, after the group."""
    processor = self.info.get_hf_processor()
    extractor = processor.feature_extractor
    seconds_per_token = (
        extractor.hop_length * extractor.encoder_stride / extractor.sampling_rate
    )
    bounds = [0] + [
        min(num_tokens, round(start / seconds_per_token))
        for start in video.group_starts[1:]
    ]
    bounds = [max(bounds[: i + 1]) for i in range(len(bounds))] + [num_tokens]
    tokens: list[int] = []
    for group, first, last in zip(video.groups, bounds, bounds[1:]):
        tokens += group + [processor.audio_token_id] * (last - first)
    return PromptUpdateDetails.select_token_id(
        self._wrap(tokens), processor.video_token_id
    )

_with_soundtrack_placeholders(prompt, num_audios)

The chat parser adds each video's decoded soundtrack as the audio item right after it; the openai content format puts no placeholder for it in the prompt. When the audio items outnumber the placeholders by exactly one per video, put that placeholder right after every video, as HF's load_audio_from_video does; otherwise the prompt is kept.

Source code in vllm/model_executor/models/lfm3_omni.py
def _with_soundtrack_placeholders(
    self, prompt: list[int], num_audios: int
) -> list[int]:
    """The chat parser adds each video's decoded soundtrack as the audio
    item right after it; the openai content format puts no placeholder for
    it in the prompt. When the audio items outnumber the placeholders by
    exactly one per video, put that placeholder right after every video,
    as HF's `load_audio_from_video` does; otherwise the prompt is kept."""
    processor = self.info.get_hf_processor()
    video_token_id = processor.video_token_id
    audio_token_id = processor.audio_token_id
    num_videos = prompt.count(video_token_id)
    if num_audios != prompt.count(audio_token_id) + num_videos:
        return prompt
    with_tracks: list[int] = []
    for token in prompt:
        with_tracks.append(token)
        if token == video_token_id:
            with_tracks.append(audio_token_id)
    return with_tracks

_VideoGroups dataclass

Bases: PromptUpdateDetails

A video placeholder that takes a soundtrack, kept as frame groups until the prompt shows which clip that is.

Source code in vllm/model_executor/models/lfm3_omni.py
@dataclass
class _VideoGroups(PromptUpdateDetails):
    """A video placeholder that takes a soundtrack, kept as frame groups until
    the prompt shows which clip that is."""

    groups: list[list[int]] = field(default_factory=list)
    group_starts: list[float] = field(default_factory=list)