Skip to content

vllm.multimodal.media.base

Classes:

  • MediaDecodedBudget –

    Thread-safe decoded-media byte accumulator for one request.

  • MediaIO –

    Configuration values can be user-provided either by --media-io-kwargs or

  • MediaWithBytes –

    Wrapper that couples a media object with its original encoded bytes.

MediaDecodedBudget

Thread-safe decoded-media byte accumulator for one request.

Methods:

  • reserve –

    Reserve decoder capacity, waiting for in-flight decodes if needed.

  • settle –

    Replace a decoder reservation with its actual decoded size.

Source code in vllm/multimodal/media/base.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
class MediaDecodedBudget:
    """Thread-safe decoded-media byte accumulator for one request."""

    def __init__(self) -> None:
        self.used = 0
        self._reserved = 0
        self._condition = threading.Condition()

    def reserve(self, max_bytes: int) -> int:
        """Reserve decoder capacity, waiting for in-flight decodes if needed."""
        limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
        if limit <= 0:
            return max_bytes
        max_bytes = max(0, max_bytes)
        with self._condition:
            while True:
                available = limit - self.used - self._reserved
                if available >= max_bytes or self._reserved == 0:
                    if available <= 0:
                        raise ValueError(
                            "Request decoded media exceeds "
                            "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
                        )
                    reserved = min(max_bytes, available)
                    self._reserved += reserved
                    return reserved
                self._condition.wait()

    def settle(self, reserved: int, decoded_bytes: int) -> None:
        """Replace a decoder reservation with its actual decoded size."""
        limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
        if limit <= 0:
            return
        with self._condition:
            self._reserved -= reserved
            try:
                used = self.used + decoded_bytes
                if decoded_bytes > reserved or used > limit:
                    raise ValueError(
                        "Request decoded media exceeds "
                        f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
                        f"({used} bytes)."
                    )
                self.used = used
            finally:
                self._condition.notify_all()

    def charge(self, decoded_bytes: int) -> None:
        limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
        if limit <= 0 or decoded_bytes <= 0:
            return
        with self._condition:
            while (
                self.used + self._reserved + decoded_bytes > limit
                and self._reserved > 0
                and self.used + decoded_bytes <= limit
            ):
                self._condition.wait()
            used = self.used + decoded_bytes
            if used > limit:
                raise ValueError(
                    "Request decoded media exceeds "
                    f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
                    f"({used} bytes)."
                )
            self.used = used

reserve(max_bytes)

Reserve decoder capacity, waiting for in-flight decodes if needed.

Source code in vllm/multimodal/media/base.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def reserve(self, max_bytes: int) -> int:
    """Reserve decoder capacity, waiting for in-flight decodes if needed."""
    limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
    if limit <= 0:
        return max_bytes
    max_bytes = max(0, max_bytes)
    with self._condition:
        while True:
            available = limit - self.used - self._reserved
            if available >= max_bytes or self._reserved == 0:
                if available <= 0:
                    raise ValueError(
                        "Request decoded media exceeds "
                        "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
                    )
                reserved = min(max_bytes, available)
                self._reserved += reserved
                return reserved
            self._condition.wait()

settle(reserved, decoded_bytes)

Replace a decoder reservation with its actual decoded size.

Source code in vllm/multimodal/media/base.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def settle(self, reserved: int, decoded_bytes: int) -> None:
    """Replace a decoder reservation with its actual decoded size."""
    limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
    if limit <= 0:
        return
    with self._condition:
        self._reserved -= reserved
        try:
            used = self.used + decoded_bytes
            if decoded_bytes > reserved or used > limit:
                raise ValueError(
                    "Request decoded media exceeds "
                    f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
                    f"({used} bytes)."
                )
            self.used = used
        finally:
            self._condition.notify_all()

MediaIO

Bases: ABC, Generic[_T]

Configuration values can be user-provided either by --media-io-kwargs or by the runtime API field "media_io_kwargs". Ensure proper validation and error handling.

Methods:

  • get_max_bytes –

    Return the maximum encoded payload size accepted by this media IO.

  • load_base64 –

    List of media types:

  • merge_kwargs –

    Merge config-level kwargs and request-level kwargs.

Source code in vllm/multimodal/media/base.py 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
class MediaIO(ABC, Generic[_T]):
    """Configuration values can be user-provided either by --media-io-kwargs or
    by the runtime API field "media_io_kwargs". Ensure proper validation and
    error handling.
    """

    @classmethod
    def merge_kwargs(
        cls,
        default_kwargs: dict[str, Any] | None,
        runtime_kwargs: dict[str, Any] | None,
    ) -> dict[str, Any]:
        """Merge config-level kwargs and request-level kwargs.

        By default this performs a shallow merge where runtime kwargs override
        keys in default kwargs. Subclasses may override to apply modality-
        specific behavior.
        """
        merged = dict(default_kwargs or {})
        if runtime_kwargs:
            merged.update(runtime_kwargs)
        return merged

    def get_max_bytes(self) -> int | None:
        """Return the maximum encoded payload size accepted by this media IO."""
        return None

    @abstractmethod
    def load_bytes(self, data: bytes) -> _T:
        raise NotImplementedError

    @abstractmethod
    def load_base64(self, media_type: str, data: str) -> _T:
        """List of media types:
        https://www.iana.org/assignments/media-types/media-types.xhtml
        """
        raise NotImplementedError

    @abstractmethod
    def load_file(self, filepath: Path) -> _T:
        raise NotImplementedError

get_max_bytes()

Return the maximum encoded payload size accepted by this media IO.

Source code in vllm/multimodal/media/base.py 
152
153
154
def get_max_bytes(self) -> int | None:
    """Return the maximum encoded payload size accepted by this media IO."""
    return None

load_base64(media_type, data) abstractmethod

List of media types: https://www.iana.org/assignments/media-types/media-types.xhtml

Source code in vllm/multimodal/media/base.py 
160
161
162
163
164
165
@abstractmethod
def load_base64(self, media_type: str, data: str) -> _T:
    """List of media types:
    https://www.iana.org/assignments/media-types/media-types.xhtml
    """
    raise NotImplementedError

merge_kwargs(default_kwargs, runtime_kwargs) classmethod

Merge config-level kwargs and request-level kwargs.

By default this performs a shallow merge where runtime kwargs override keys in default kwargs. Subclasses may override to apply modality- specific behavior.

Source code in vllm/multimodal/media/base.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
@classmethod
def merge_kwargs(
    cls,
    default_kwargs: dict[str, Any] | None,
    runtime_kwargs: dict[str, Any] | None,
) -> dict[str, Any]:
    """Merge config-level kwargs and request-level kwargs.

    By default this performs a shallow merge where runtime kwargs override
    keys in default kwargs. Subclasses may override to apply modality-
    specific behavior.
    """
    merged = dict(default_kwargs or {})
    if runtime_kwargs:
        merged.update(runtime_kwargs)
    return merged

MediaWithBytes dataclass

Bases: Generic[_T]

Wrapper that couples a media object with its original encoded bytes.

This ensures the raw bytes and media object remain synchronized, preventing cache corruption from in-place modifications.

The wrapper delegates attribute access to the underlying media object, making it behave transparently like the wrapped type (e.g., PIL.Image).

NOTE: Currently, this wrapper is used only for the image and video modalities.

Methods:

  • __array__ –

    Allow np.array(obj) to return np.array(obj.media).

  • __getattr__ –

    Delegate attribute access to the underlying media object.

  • __getitem__ –

    Allow obj[i] to index obj.media (e.g. video tuples).

  • __iter__ –

    Allow unpacking obj to unpack obj.media (e.g. video tuples).

Attributes:

  • io_config (dict[str, Any] | None) –

    Decode settings that altered the media relative to original_bytes

Source code in vllm/multimodal/media/base.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
@dataclass
class MediaWithBytes(Generic[_T]):
    """Wrapper that couples a media object with its original encoded bytes.

    This ensures the raw bytes and media object remain synchronized,
    preventing cache corruption from in-place modifications.

    The wrapper delegates attribute access to the underlying media object,
    making it behave transparently like the wrapped type (e.g., PIL.Image).

    NOTE: Currently, this wrapper is used only for the image and video
    modalities.
    """

    media: _T
    original_bytes: bytes = field(repr=False)
    io_config: dict[str, Any] | None = None
    """Decode settings that altered the media relative to `original_bytes`
    (e.g. `image_mode` conversion), so they participate in cache hashing."""

    def __array__(self, *args, **kwargs) -> np.ndarray:
        """Allow np.array(obj) to return np.array(obj.media)."""
        return np.array(self.media, *args, **kwargs)

    def __iter__(self) -> Iterator[Any]:
        """Allow unpacking obj to unpack obj.media (e.g. video tuples)."""
        return iter(cast(Iterable[Any], self.media))

    def __getitem__(self, index: Any) -> Any:
        """Allow obj[i] to index obj.media (e.g. video tuples)."""
        return cast(Any, self.media)[index]

    def __getstate__(self):
        return self.__dict__.copy()

    def __setstate__(self, state: dict[str, Any]):
        self.__dict__.update(state)

    def __getattr__(self, name: str):
        """Delegate attribute access to the underlying media object."""
        return getattr(self.media, name)

io_config = None class-attribute instance-attribute

Decode settings that altered the media relative to original_bytes (e.g. image_mode conversion), so they participate in cache hashing.

__array__(*args, **kwargs)

Allow np.array(obj) to return np.array(obj.media).

Source code in vllm/multimodal/media/base.py 
106
107
108
def __array__(self, *args, **kwargs) -> np.ndarray:
    """Allow np.array(obj) to return np.array(obj.media)."""
    return np.array(self.media, *args, **kwargs)

__getattr__(name)

Delegate attribute access to the underlying media object.

Source code in vllm/multimodal/media/base.py 
124
125
126
def __getattr__(self, name: str):
    """Delegate attribute access to the underlying media object."""
    return getattr(self.media, name)

__getitem__(index)

Allow obj[i] to index obj.media (e.g. video tuples).

Source code in vllm/multimodal/media/base.py 
114
115
116
def __getitem__(self, index: Any) -> Any:
    """Allow obj[i] to index obj.media (e.g. video tuples)."""
    return cast(Any, self.media)[index]

__iter__()

Allow unpacking obj to unpack obj.media (e.g. video tuples).

Source code in vllm/multimodal/media/base.py 
110
111
112
def __iter__(self) -> Iterator[Any]:
    """Allow unpacking obj to unpack obj.media (e.g. video tuples)."""
    return iter(cast(Iterable[Any], self.media))