[OpenVINO] Support Youtu-VL-4B-Instruct with task image-text-to-text

docs/source/openvino/models.mdx

@@ -175,6 +175,7 @@ Here is the list of the supported architectures :
 - XLM
 - XLM-RoBERTa
 - XVERSE
+- Youtu-VL
 - Zamba2
 
 ## [Diffusers](https://huggingface.co/docs/diffusers/index)

optimum/exporters/openvino/input_generators.py

@@ -1396,6 +1396,103 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
             )
 
 
+class YoutuVLDummyVisionEmbedInputGenerator(DummyVisionInputGenerator):
+    """Dummy input generator for the youtu_vl vision embeddings submodel.
+
+    The youtu_vl vision tower is a siglip2-based encoder that consumes already-patchified pixel
+    values of shape (batch_size, num_patches, num_channels * patch_size * patch_size).
+    """
+
+    SUPPORTED_INPUT_NAMES = ("pixel_values",)
+
+    def __init__(
+        self,
+        task: str,
+        normalized_config: NormalizedVisionConfig,
+        batch_size: int = 1,
+        num_channels: int = DEFAULT_DUMMY_SHAPES["num_channels"],
+        width: int = 64,
+        height: int = 64,
+        **kwargs,
+    ):
+        self.task = task
+        self.batch_size = batch_size
+        self.num_channels = normalized_config.config.num_channels
+        self.patch_size = normalized_config.config.patch_size
+        in_features = getattr(normalized_config.config, "in_features", -1)
+        if in_features is not None and in_features > 0:
+            self.in_features = in_features
+        else:
+            self.in_features = self.num_channels * self.patch_size * self.patch_size
+        spatial_merge_size = getattr(normalized_config.config, "spatial_merge_size", 2)
+        grid = max(spatial_merge_size * 2, 4)
+        self.num_patches = grid * grid
+
+    def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
+        if input_name == "pixel_values":
+            return self.random_float_tensor(
+                [self.batch_size, self.num_patches, self.in_features], framework=framework, dtype=float_dtype
+            )
+
+
+class YoutuVLDummyVisionMergerInputGenerator(DummyVisionInputGenerator):
+    """Dummy input generator for the youtu_vl vision embeddings merger submodel.
+
+    Mirrors the Qwen2.5-VL windowed-attention merger inputs: encoder hidden states plus the
+    precomputed full/window attention masks, window index and rotary position embeddings.
+    """
+
+    SUPPORTED_INPUT_NAMES = (
+        "hidden_states",
+        "attention_mask",
+        "window_attention_mask",
+        "window_index",
+        "rotary_pos_emb",
+    )
+
+    def __init__(
+        self,
+        task: str,
+        normalized_config: NormalizedVisionConfig,
+        batch_size: int = 1,
+        num_channels: int = DEFAULT_DUMMY_SHAPES["num_channels"],
+        width: int = 64,
+        height: int = 64,
+        **kwargs,
+    ):
+        self.task = task
+        self.batch_size = batch_size
+        self.hidden_size = normalized_config.config.hidden_size
+        self.num_heads = normalized_config.config.num_attention_heads
+        self.spatial_merge_size = getattr(normalized_config.config, "spatial_merge_size", 2)
+        grid = max(self.spatial_merge_size * 2, 4)
+        self.seq_len = grid * grid
+
+    def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
+        if input_name == "hidden_states":
+            return self.random_float_tensor([self.seq_len, self.hidden_size], framework=framework, dtype=float_dtype)
+
+        if input_name in ["attention_mask", "window_attention_mask"]:
+            return self.random_mask_tensor([1, self.seq_len, self.seq_len], framework=framework, dtype=float_dtype)
+
+        if input_name == "rotary_pos_emb":
+            dim = self.hidden_size // self.num_heads // 2
+            return self.random_float_tensor([self.seq_len, dim], framework=framework, dtype=float_dtype)
+
+        if input_name == "window_index":
+            spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
+            length = self.seq_len // spatial_merge_unit
+            return self.random_int_tensor([length], max_value=length)
+
+
+class YoutuVLDummyPastKeyValuesGenerator(OVMiniCPM3DummyPastKeyValuesGenerator):
+    """Past-key-values generator for the youtu_vl MLA (multi-head latent attention) language model.
+
+    The cache stores per-head keys of dim (qk_nope_head_dim + qk_rope_head_dim) and values of
+    dim v_head_dim, identical in layout to the MiniCPM3/DeepSeek-V2 MLA cache.
+    """
+
+
 class Qwen3ASRDummySeq2SeqPastKeyValuesGenerator(DummySeq2SeqPastKeyValuesGenerator):
     """Custom KV cache generator for Qwen3-ASR with GQA (num_key_value_heads != num_attention_heads).
     Qwen3-ASR has no cross-attention, so only self-attention KV cache is generated (2 per layer)."""

optimum/exporters/openvino/model_configs.py

@@ -93,6 +93,9 @@
     Qwen3ASRDummySeq2SeqPastKeyValuesGenerator,
     Qwen3NextDummyPastKeyValuesGenerator,
     QwenDummyPastKeyValuesGenerator,
+    YoutuVLDummyPastKeyValuesGenerator,
+    YoutuVLDummyVisionEmbedInputGenerator,
+    YoutuVLDummyVisionMergerInputGenerator,
     Zamba2DummyPastKeyValuesGenerator,
 )
 from optimum.exporters.openvino.model_patcher import (
@@ -187,6 +190,9 @@
     SpeechT5ModelPatcher,
     VideoChatFlashQwenVisionEmbeddingModelPatcher,
     XverseModelPatcher,
+    YoutuVLLanguageModelPatcher,
+    YoutuVLModelPatcher,
+    YoutuVLVisionEmbMergerPatcher,
     Zamba2ModelPatcher,
     _get_model_attribute,
 )
@@ -3595,6 +3601,146 @@ def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[
         return super().patch_model_for_export(model, model_kwargs)
 
 
+@register_in_tasks_manager("youtu_vl", *["image-text-to-text"], library_name="transformers")
+class YoutuVLOpenVINOConfig(BaseVLMOpenVINOConfig):
+    SUPPORTED_BEHAVIORS = [
+        model_type.value for model_type in QwenVLConfigBehavior if model_type.value != "vision_embeddings_pos"
+    ]
+    NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig
+    DUMMY_INPUT_GENERATOR_CLASSES = (YoutuVLDummyVisionEmbedInputGenerator,)
+    MIN_TRANSFORMERS_VERSION = "4.57"
+
+    def __init__(
+        self,
+        config: "PretrainedConfig",
+        task: str = "feature-extraction",
+        int_dtype: str = "int64",
+        float_dtype: str = "fp32",
+        behavior: QwenVLConfigBehavior = QwenVLConfigBehavior.VISION_EMBEDDINGS,
+        preprocessors: Optional[List[Any]] = None,
+    ):
+        super().__init__(
+            config=config,
+            task=task,
+            int_dtype=int_dtype,
+            float_dtype=float_dtype,
+            preprocessors=preprocessors,
+        )
+        self._behavior = behavior
+        self._orig_config = config
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS and hasattr(config, "vision_config"):
+            self._config = config.vision_config
+            self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config)
+            self.DUMMY_INPUT_GENERATOR_CLASSES = (YoutuVLDummyVisionEmbedInputGenerator,)
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER and hasattr(config, "vision_config"):
+            self._config = config.vision_config
+            self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config)
+            self.DUMMY_INPUT_GENERATOR_CLASSES = (YoutuVLDummyVisionMergerInputGenerator,)
+
+    @staticmethod
+    def get_model_for_behavior(model, behavior: Union[str, QwenVLConfigBehavior]):
+        if isinstance(behavior, str) and not isinstance(behavior, QwenVLConfigBehavior):
+            behavior = QwenVLConfigBehavior(behavior)
+
+        if behavior == QwenVLConfigBehavior.LANGUAGE:
+            return model
+
+        if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS:
+            vision_embeddings = model.siglip2.vision_model.embeddings
+            vision_embeddings.config = model.config.vision_config
+            return vision_embeddings
+
+        if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            # The merger patcher runs the siglip2 encoder + post_layernorm + patch merger, all of
+            # which are reachable from the top-level model, so we return the whole model here.
+            model.config.vision_config.torchscript = True
+            return model
+
+        if behavior == QwenVLConfigBehavior.TEXT_EMBEDDINGS:
+            text_embedding = model.model.embed_tokens
+            text_embedding.config = model.config
+            return text_embedding
+
+    def with_behavior(
+        self,
+        behavior: Union[str, QwenVLConfigBehavior],
+    ):
+        """
+        Creates a config for different behaviour.
+        Args:
+            behavior ([`ConfigBehavior`]):
+                The behavior to use for the new instance.
+        """
+        if isinstance(behavior, str) and not isinstance(behavior, QwenVLConfigBehavior):
+            behavior = QwenVLConfigBehavior(behavior)
+
+        if behavior == QwenVLConfigBehavior.TEXT_EMBEDDINGS:
+            return get_vlm_text_embeddings_config(
+                "youtu_vl_text",
+                self._orig_config,
+                self.int_dtype,
+                self.float_dtype,
+            )
+
+        if behavior == QwenVLConfigBehavior.LANGUAGE:
+            return get_vlm_text_generation_config(
+                "youtu_vl_text",
+                self._orig_config,
+                self.int_dtype,
+                self.float_dtype,
+                model_patcher=YoutuVLLanguageModelPatcher,
+            )
+
+        if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS:
+            return self.__class__(
+                self._orig_config,
+                task=self.task,
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+                behavior=behavior,
+                preprocessors=self._preprocessors,
+            )
+        if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return self.__class__(
+                self._orig_config,
+                task=self.task,
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+                behavior=behavior,
+                preprocessors=self._preprocessors,
+            )
+
+    def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[Dict[str, Any]] = None):
+        model_kwargs = model_kwargs or {}
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return YoutuVLVisionEmbMergerPatcher(self, model, model_kwargs)
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS:
+            return ModelPatcher(self, model, model_kwargs=model_kwargs)
+        return super().patch_model_for_export(model, model_kwargs)
+
+    @property
+    def inputs(self) -> Dict[str, Dict[int, str]]:
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS:
+            return {"pixel_values": {0: "batch_size", 1: "num_patches"}}
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return {
+                "hidden_states": {0: "sequence_length"},
+                "attention_mask": {1: "sequence_length", 2: "sequence_length"},
+                "window_attention_mask": {1: "sequence_length", 2: "sequence_length"},
+                "window_index": {0: "unit_sequence_length"},
+                "rotary_pos_emb": {0: "sequence_length"},
+            }
+        return {}
+
+    @property
+    def outputs(self) -> Dict[str, Dict[int, str]]:
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS:
+            return {"last_hidden_state": {0: "sequence_length"}}
+        if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return {"last_hidden_state": {0: "seq_len"}}
+        return {}
+
+
 @register_in_tasks_manager(
     "qwen3_vl",
     *[
@@ -4587,6 +4733,20 @@ class DeepseekOpenVINOConfig(MiniCPM3OpenVINOConfig):
     _MODEL_PATCHER = DeepseekPatcher
 
 
+@register_in_tasks_manager(
+    "youtu_vl_text", *["text-generation", "text-generation-with-past"], library_name="transformers"
+)
+class YoutuVLTextOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig):
+    # youtu_vl language model: DeepSeek-V2 style multi-head latent attention (MLA) with a standard
+    # (dense) MLP. Its KV cache stores per-head keys of dim (qk_nope_head_dim + qk_rope_head_dim)
+    # and values of dim v_head_dim, identical in layout to the MiniCPM3/DeepSeek-V2 MLA cache.
+    MIN_TRANSFORMERS_VERSION = "4.57"
+    DUMMY_INPUT_GENERATOR_CLASSES = (DummyTextInputGenerator, YoutuVLDummyPastKeyValuesGenerator)
+    DUMMY_PKV_GENERATOR_CLASS = YoutuVLDummyPastKeyValuesGenerator
+    NORMALIZED_CONFIG_CLASS = NormalizedTextConfig
+    _MODEL_PATCHER = YoutuVLModelPatcher
+
+
 @register_in_tasks_manager("got_ocr2", *["image-to-text", "image-text-to-text"], library_name="transformers")
 class GotOCR2OpenVINOConfig(BaseVLMOpenVINOConfig):
     MAX_TRANSFORMERS_VERSION = "4.57.6"