Skip to content

vllm.model_executor.models.clef_omni

The Qwen3-Omni thinker with a Clef joint schema head.

Classes:

Functions:

  • cap_frame_pixels –

    The reference reads each video frame at at most 256 * 32 * 32 pixels,

ClefOmniMultiModalProcessor

Bases: Qwen3OmniMoeThinkerMultiModalProcessor

Caps video frames at the reference's per-frame pixel count.

Source code in vllm/model_executor/models/clef_omni.py 
61
62
63
64
65
66
67
68
69
70
71
72
class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
    """Caps video frames at the reference's per-frame pixel count."""

    def _call_hf_processor(
        self,
        hf_data: Mapping[str, object],
        hf_kwargs: Mapping[str, object],
    ) -> BatchFeature:
        if videos := hf_data.get("videos"):
            assert isinstance(videos, list)
            hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
        return super()._call_hf_processor(hf_data, hf_kwargs)

cap_frame_pixels(frames)

The reference reads each video frame at at most 256 * 32 * 32 pixels, in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on their device (a CUDA tensor from torchcodec must not round-trip).

Source code in vllm/model_executor/models/clef_omni.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
    """The reference reads each video frame at at most 256 * 32 * 32 pixels,
    in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
    their device (a CUDA tensor from torchcodec must not round-trip)."""
    height, width = frames.shape[1:3]
    if height * width <= MAX_FRAME_PIXELS:
        return frames
    scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
    new_width = max(2, round(width * scale / 2) * 2)
    new_height = max(2, round(height * scale / 2) * 2)
    is_tensor = isinstance(frames, torch.Tensor)
    tensor = frames if is_tensor else torch.from_numpy(frames)
    resized = F.interpolate(
        tensor.permute(0, 3, 1, 2).float(),
        size=(new_height, new_width),
        mode="bilinear",
    )
    if tensor.dtype == torch.uint8:
        resized = resized.round()
    resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
    return resized if is_tensor else resized.numpy()