diff --git a/docs/source/openvino/models.mdx b/docs/source/openvino/models.mdx index 902dd2ad75..13ad8ad471 100644 --- a/docs/source/openvino/models.mdx +++ b/docs/source/openvino/models.mdx @@ -110,6 +110,7 @@ Here is the list of the supported architectures : - MiniCPM3 - MiniCPM-o - MiniCPM-V +- MiniCPM-V-4.6 - Mistral - Mixtral - MobileBERT diff --git a/optimum/exporters/openvino/input_generators.py b/optimum/exporters/openvino/input_generators.py index 34f75923ba..b01986fb2e 100644 --- a/optimum/exporters/openvino/input_generators.py +++ b/optimum/exporters/openvino/input_generators.py @@ -1116,6 +1116,102 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int return self.random_float_tensor(shape=[self.feat_size, self.batch_size, self.hidden_size]) +class DummyMiniCPMV4_6ImageInputGenerator(DummyVisionInputGenerator): + """Dummy inputs for the fully-fused MiniCPM-V-4.6 image feature extractor. + + The exported graph takes the NaViT-packed ``pixel_values`` together with a set of + precomputed index / mask tensors (patch position ids, block-diagonal encoder and + window attention masks, window reordering indices, and spatial-merge gather + indices). This generator builds a self-consistent set for a small square dummy + grid so that all divisibility invariants (window 2x2 then merge 2x2) hold. + """ + + SUPPORTED_INPUT_NAMES = ( + "pixel_values", + "pos_ids", + "encoder_attention_mask", + "downsampled_attention_mask", + "window_index", + "reverse_window_index", + "window_attention_mask", + "merge_gather_index", + "final_gather_index", + ) + + def __init__( + self, + task: str, + normalized_config: NormalizedVisionConfig, + batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"], + num_channels: int = DEFAULT_DUMMY_SHAPES["num_channels"], + width: int = DEFAULT_DUMMY_SHAPES["width"], + height: int = DEFAULT_DUMMY_SHAPES["height"], + **kwargs, + ): + super().__init__(task, normalized_config, batch_size, num_channels, width, height) + vision_config = normalized_config.config + self.patch_size = vision_config.patch_size + # Small square grid (8x8 patches) that stays divisible by the window (2x2) + # and, after the /2 window merge, by the merge kernel (2x2). + self.grid_h = 8 + self.grid_w = 8 + self.window_h, self.window_w = 2, 2 + self.merge_h, self.merge_w = 2, 2 + self.num_patches = self.grid_h * self.grid_w + self.merged_h = self.grid_h // self.window_h + self.merged_w = self.grid_w // self.window_w + self.merged_patches = self.merged_h * self.merged_w + self.final_h = self.merged_h // self.merge_h + self.final_w = self.merged_w // self.merge_w + self.final_patches = self.final_h * self.final_w + + def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): + if input_name == "pixel_values": + return self.random_float_tensor( + shape=[1, self.num_channels, self.patch_size, self.num_patches * self.patch_size], + framework=framework, + dtype=float_dtype, + ) + if input_name == "pos_ids": + return self.random_int_tensor( + shape=[self.num_patches], min_value=0, max_value=8, framework=framework, dtype=int_dtype + ) + if input_name in ("encoder_attention_mask", "window_attention_mask"): + return self.constant_tensor( + shape=[1, self.num_patches, self.num_patches], + value=0.0, + framework=framework, + dtype=DTYPE_MAPPER.pt(float_dtype), + ) + if input_name == "downsampled_attention_mask": + return self.constant_tensor( + shape=[1, self.merged_patches, self.merged_patches], + value=0.0, + framework=framework, + dtype=DTYPE_MAPPER.pt(float_dtype), + ) + if input_name in ("window_index", "reverse_window_index"): + return self.random_int_tensor( + shape=[self.num_patches], min_value=0, max_value=self.num_patches, framework=framework, dtype=int_dtype + ) + if input_name == "merge_gather_index": + return self.random_int_tensor( + shape=[self.merged_patches * self.window_h * self.window_w], + min_value=0, + max_value=self.num_patches, + framework=framework, + dtype=int_dtype, + ) + if input_name == "final_gather_index": + return self.random_int_tensor( + shape=[self.final_patches * self.merge_h * self.merge_w], + min_value=0, + max_value=self.merged_patches, + framework=framework, + dtype=int_dtype, + ) + + class DummyPhi3VisionProjectionInputGenerator(DummyVisionInputGenerator): SUPPORTED_INPUT_NAMES = ("input",) diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 1bb72ff0e4..e4bec3e3f6 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -47,6 +47,7 @@ DummyGemma4VisionInputGenerator, DummyKokoroInputGenerator, DummyLLavaMultiModalProjectorInputGenerator, + DummyMiniCPMV4_6ImageInputGenerator, DummyMiniCPMVImageInputGenerator, DummyMiniCPMVResampleInputGenerator, DummyPhi3VisionProjectionInputGenerator, @@ -147,6 +148,7 @@ MambaPatcher, MiniCPM3Patcher, MiniCPMModelPatcher, + MiniCPMV4_6VisionEmbeddingsModelPatcher, MiniCPMVImageEmbeddingsModelPatcher, MiniCPMVResamplerModelPatcher, MistralModelPatcher, @@ -6895,6 +6897,142 @@ def outputs(self) -> Dict[str, Dict[int, str]]: return super().outputs +class MiniCPMV4_6ConfigBehavior(str, enum.Enum): + LANGUAGE = "language" + VISION_EMBEDDINGS = "vision_embeddings" + TEXT_EMBEDDINGS = "text_embeddings" + + +@register_in_tasks_manager("minicpmv4_6", *["image-text-to-text"], library_name="transformers") +class MiniCPMV4_6OpenVINOConfig(BaseVLMOpenVINOConfig): + """OpenVINO exporter configuration for MiniCPM-V-4.6. + + MiniCPM-V-4.6 combines a NaViT-packed SigLIP-style vision encoder with a ViT + window-attention merger + downsample merger, and a ``qwen3_5_text`` hybrid + (linear + full attention) language backbone. Unlike the older resampler-based + ``minicpmv`` architecture, image features are inserted through a simple + ``masked_scatter`` on ``image_token_id`` and the text backbone uses standard + 1D RoPE position ids (its ``rope_type`` is ``default`` with no mrope-section + effect), so no 3D position handling is required. + """ + + SUPPORTED_BEHAVIORS = [model_type.value for model_type in MiniCPMV4_6ConfigBehavior] + NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig + DUMMY_INPUT_GENERATOR_CLASSES = () + MIN_TRANSFORMERS_VERSION = "5.7.0" + MODEL_TYPE = "minicpmv4_6" + + def __init__( + self, + config: "PretrainedConfig", + task: str = "feature-extraction", + int_dtype: str = "int64", + float_dtype: str = "fp32", + behavior: MiniCPMV4_6ConfigBehavior = MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS, + preprocessors: Optional[List[Any]] = None, + ): + super().__init__( + config=config, + task=task, + int_dtype=int_dtype, + float_dtype=float_dtype, + preprocessors=preprocessors, + behavior=behavior, + ) + self._behavior = behavior + self._orig_config = config + if self._behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS and hasattr(config, "vision_config"): + self._config = config.vision_config + self.DUMMY_INPUT_GENERATOR_CLASSES = (DummyMiniCPMV4_6ImageInputGenerator,) + self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config) + + @property + def inputs(self) -> Dict[str, Dict[int, str]]: + if self._behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS: + return { + "pixel_values": {0: "batch_size", 3: "patch_seq"}, + "pos_ids": {0: "num_patches"}, + "encoder_attention_mask": {1: "num_patches", 2: "num_patches"}, + "downsampled_attention_mask": {1: "merged_patches", 2: "merged_patches"}, + "window_index": {0: "num_patches"}, + "reverse_window_index": {0: "num_patches"}, + "window_attention_mask": {1: "num_patches", 2: "num_patches"}, + "merge_gather_index": {0: "merge_gather"}, + "final_gather_index": {0: "final_gather"}, + } + return {} + + @property + def outputs(self) -> Dict[str, Dict[int, str]]: + if self._behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS: + return {"image_features": {0: "num_image_tokens"}} + return {} + + def with_behavior( + self, + behavior: Union[str, MiniCPMV4_6ConfigBehavior], + ): + if isinstance(behavior, str) and not isinstance(behavior, MiniCPMV4_6ConfigBehavior): + behavior = MiniCPMV4_6ConfigBehavior(behavior) + + if behavior == MiniCPMV4_6ConfigBehavior.TEXT_EMBEDDINGS: + return get_vlm_text_embeddings_config( + "qwen3_5_text", + self._orig_config.text_config, + self.int_dtype, + self.float_dtype, + min_transformers_version=self.MIN_TRANSFORMERS_VERSION, + ) + + if behavior == MiniCPMV4_6ConfigBehavior.LANGUAGE: + # MiniCPM-V-4.6 feeds the qwen3_5_text backbone standard 1D (2D + # batched) position ids — its rope_type is ``default`` with no mrope + # section, so the 3D mrope position ids used by the standalone Qwen3.5 + # VLM are not required here. + return get_vlm_text_generation_config( + "qwen3_5_text", + self._orig_config.text_config, + self.int_dtype, + self.float_dtype, + model_patcher=Qwen3_5ModelPatcher, + min_transformers_version=self.MIN_TRANSFORMERS_VERSION, + ) + + if behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS: + return self.__class__( + self._orig_config, + task=self.task, + int_dtype=self.int_dtype, + float_dtype=self.float_dtype, + behavior=behavior, + preprocessors=self._preprocessors, + ) + + @staticmethod + def get_model_for_behavior(model, behavior: Union[str, MiniCPMV4_6ConfigBehavior]): + if isinstance(behavior, str) and not isinstance(behavior, MiniCPMV4_6ConfigBehavior): + behavior = MiniCPMV4_6ConfigBehavior(behavior) + + if behavior == MiniCPMV4_6ConfigBehavior.LANGUAGE: + return model + + if behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS: + # top-level MiniCPMV4_6Model so the patcher can reach both the vision + # tower (with its window merger) and the downsample merger. + return model.model + + if behavior == MiniCPMV4_6ConfigBehavior.TEXT_EMBEDDINGS: + text_embedding = model.model.get_input_embeddings() + text_embedding.config = model.model.language_model.config + return text_embedding + + def patch_model_for_export(self, model: "PreTrainedModel", model_kwargs: Optional[Dict[str, Any]] = None): + model_kwargs = model_kwargs or {} + if self._behavior == MiniCPMV4_6ConfigBehavior.VISION_EMBEDDINGS: + return MiniCPMV4_6VisionEmbeddingsModelPatcher(self, model, model_kwargs) + return super().patch_model_for_export(model, model_kwargs) + + @register_in_tasks_manager( "kokoro", *["text-to-audio"], diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index a93cf65a7d..272543caf8 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -9479,7 +9479,7 @@ def __init__( model: "PreTrainedModel", model_kwargs: Optional[Dict[str, Any]] = None, ): - from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DynamicCache + from transformers.cache_utils import Cache from openvino.frontend.pytorch import ConversionExtension, ModuleExtension @@ -9496,26 +9496,39 @@ def __init__( self._text_model = self._model.model self._text_config = self._model.model.config - class Qwen3_5DynamicCacheWrap(Qwen3_5DynamicCache): + # NOTE: In transformers < 5.6 the Qwen3.5 backbone shipped a dedicated + # ``Qwen3_5DynamicCache`` class. Starting with transformers 5.6/5.7 the + # hybrid (linear + full attention) cache is expressed through the generic + # ``DynamicCache`` composed of per-layer cache mixins, and the dedicated + # class was removed. This wrapper reimplements only the pieces the export + # graph needs (flat conv/ssm/key/value state lists indexed by their + # per-kind position, plus ``update``/``get_seq_length``/``has_previous_state``) + # so it stays compatible with both cache generations. The linear-attention + # layers are fully replaced by ``qwen3_5_gated_delta_net_forward`` which + # reads/writes ``conv_states``/``recurrent_states`` directly, while the + # full-attention layers call ``update``. + class Qwen3_5DynamicCacheWrap(Cache): def __init__(self, config, conv_states, recurrent_states, key_cache, value_cache): - # Call parent constructor with all required arguments - super().__init__(config=config) - + self.layer_types = list(config.layer_types) self.conv_states = conv_states self.recurrent_states = recurrent_states self.key_cache = key_cache self.value_cache = value_cache self.full_attn_mapping = {} self.linear_attn_mapping = {} + self.transformer_layers = [] + self.last_linear_layer = -1 full_attn_layer_idx = 0 linear_attn_layer_idx = 0 - for i in range(len(config.layer_types)): + for i in range(len(self.layer_types)): if self.layer_types[i] == "full_attention": self.full_attn_mapping[i] = full_attn_layer_idx full_attn_layer_idx += 1 + self.transformer_layers.append(i) elif self.layer_types[i] == "linear_attention": self.linear_attn_mapping[i] = linear_attn_layer_idx linear_attn_layer_idx += 1 + self.last_linear_layer = i def update( self, @@ -9544,12 +9557,44 @@ def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - @property - def has_previous_state(self): - """We have a previous state if the last linear (conv) layer was already updated.""" + def has_previous_state(self, layer_idx: Optional[int] = None): + """We have a previous state if the last linear (conv) layer was already updated. + + transformers < 5.6 accessed this as a property, while transformers >= 5.6 + calls it as a method (optionally per-layer). Implemented as a method that + ignores ``layer_idx`` to stay compatible with both cache generations. + """ + if self.last_linear_layer < 0: + return False layer_idx = self.linear_attn_mapping[self.last_linear_layer] return self.conv_states[layer_idx] is not None + def get_mask_sizes(self, query_length: int, layer_idx: int = 0) -> tuple: + """Return ``(kv_length, kv_offset)`` for the causal-mask machinery. + + transformers >= 5.6 asks the cache for the key/value length through + ``get_mask_sizes`` (previously it read the KV tensors directly). This + wrapper stores the full-attention key cache in ``key_cache`` indexed by + full-attention position, so derive the past length from the first + available full-attention layer. + + Following the canonical ``DynamicLayer.get_mask_sizes`` contract, the + full-attention keys always start at absolute position 0, so ``kv_offset`` + must be 0 and ``kv_length`` is the total number of cached keys plus the + current query length. The query's absolute offset is provided separately + by ``get_seq_length`` in ``create_causal_mask``. Returning + ``kv_offset = past_length`` here would shift the key positions past the + query and mask out every previously cached token during decode. + """ + past_length = 0 + for kv in self.key_cache: + if kv is not None: + past_length = kv.shape[-2] + break + kv_length = query_length + past_length + kv_offset = 0 + return kv_length, kv_offset + # the patch is needed to include KV-cache, Conv, and SSM states in the inputs and outputs. def patched_forward( input_ids=None, @@ -9695,6 +9740,129 @@ def __exit__(self, exc_type, exc_value, traceback): block.attn.forward = block.attn._orig_forward +def _minicpmv4_6_vision_attention(attn, hidden_states, attention_mask): + """Trace-friendly dense attention for the MiniCPM-V-4.6 vision encoder / window merger. + + The upstream ``MiniCPMV4_6VisionAttention`` uses ``cu_seqlens`` to split the + NaViT-packed sequence into per-image (or per-window) chunks and runs + attention on each chunk. That data-dependent splitting cannot be traced, so + the OpenVINO runtime precomputes an additive block-diagonal ``attention_mask`` + (0 inside a chunk, ``-inf`` across chunks) and this helper runs a single dense + attention with that mask, which is numerically identical. + + Original PyTorch code (``MiniCPMV4_6VisionAttention.forward``): + https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py + """ + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, attn.head_dim) + query_states = attn.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = attn.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = attn.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * attn.scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states).transpose(1, 2).reshape(*input_shape, -1) + return attn.out_proj(attn_output) + + +class MiniCPMV4_6VisionEmbeddingsModelPatcher(ModelPatcher): + """Exports the full MiniCPM-V-4.6 image feature extractor (NaViT SigLIP encoder, + ViT window-attention merger, and downsample merger) into a single traceable graph. + + All data-dependent index / mask computation (NaViT packing ``cu_seqlens``, + patch position ids, window reordering indices, and spatial-merge gather indices) + is precomputed on the Python side by ``_OVMiniCPMV4_6ForCausalLM`` and passed in + as plain tensors, so the graph itself only contains fixed tensor ops. + + Traced forward reproduces the upstream ``MiniCPMV4_6VisionModel.forward`` + + ``MiniCPMV4_6ViTWindowAttentionMerger.forward`` + ``MiniCPMV4_6Merger.forward``: + https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py + """ + + def __init__( + self, + config: "OpenVINOConfig", + model: "PreTrainedModel", + model_kwargs: Dict[str, Any] = None, + ): + model.__orig_forward = model.forward + + # ``model`` here is the top-level MiniCPMV4_6Model so that we can reach the + # vision tower (with its vit_merger) and the downsample merger together. + def image_features_forward( + self, + pixel_values, + pos_ids, + encoder_attention_mask, + downsampled_attention_mask, + window_index, + reverse_window_index, + window_attention_mask, + merge_gather_index, + final_gather_index, + ): + vt = self.vision_tower + merger = vt.vit_merger + window_h, window_w = merger.window_kernel_size + merge_h, merge_w = self.merger.merge_kernel_size + + hidden_states = vt.embeddings.patch_embedding(pixel_values).flatten(2).transpose(1, 2) + hidden_states = hidden_states + vt.embeddings.position_embedding(pos_ids).unsqueeze(0) + + embed_dim = hidden_states.shape[-1] + insert_layer_id = vt.config.insert_layer_id + for layer_index, encoder_layer in enumerate(vt.encoder.layers): + mask = encoder_attention_mask if layer_index <= insert_layer_id else downsampled_attention_mask + residual = hidden_states + hidden_states = encoder_layer.layer_norm1(hidden_states) + hidden_states = _minicpmv4_6_vision_attention(encoder_layer.self_attn, hidden_states, mask) + hidden_states = residual + hidden_states + residual = hidden_states + hidden_states = encoder_layer.layer_norm2(hidden_states) + hidden_states = encoder_layer.mlp(hidden_states) + hidden_states = residual + hidden_states + + if layer_index == insert_layer_id: + residual = hidden_states + windowed = merger.layer_norm1(hidden_states) + windowed = windowed[:, window_index, :] + windowed = _minicpmv4_6_vision_attention(merger.self_attn, windowed, window_attention_mask) + windowed = windowed[:, reverse_window_index, :] + hidden_states = residual + windowed + + # NaViT packs one or more image tiles (each with its own grid) into a + # single sequence. ``merge_gather_index`` reorders patches so that every + # ``window_h * window_w`` block is contiguous, so a ``-1`` row dimension + # collapses all tiles at once and is independent of the per-tile grid. + patch = hidden_states[0] + gathered = patch[merge_gather_index] + merged = gathered.reshape(-1, window_h * window_w * embed_dim) + merge_residual = gathered.reshape(-1, window_h * window_w, embed_dim).mean(dim=1) + merged = merger.pre_norm(merged) + merged = merger.linear_1(merged) + merged = merger.act(merged) + merged = merger.linear_2(merged) + hidden_states = (merged + merge_residual).unsqueeze(0) + + hidden_states = vt.post_layernorm(hidden_states) + + patch = hidden_states[0] + final_embed_dim = patch.shape[-1] + gathered = patch[final_gather_index] + merged = gathered.reshape(-1, merge_h * merge_w * final_embed_dim) + image_features = self.merger.mlp[0](merged) + return image_features + + model.forward = types.MethodType(image_features_forward, model) + super().__init__(config, model, model_kwargs) + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + self._model.forward = self._model.__orig_forward + + # Patched forward for MobileNetV5MultiScaleFusionAdapter (MSFA) used by the Gemma3n vision tower. # The original MSFA forward has data-dependent control flow that branches on tensor spatial # dimensions to choose between F.interpolate and F.avg_pool2d for resizing to output_resolution. diff --git a/optimum/exporters/openvino/utils.py b/optimum/exporters/openvino/utils.py index 55c6d852e2..6b5cdd53c9 100644 --- a/optimum/exporters/openvino/utils.py +++ b/optimum/exporters/openvino/utils.py @@ -320,6 +320,7 @@ def _get_kokoro_submodels(model): "internvl_chat", "maira2", "minicpmv", + "minicpmv4_6", "phi3_v", "qwen2_vl", "qwen2_5_vl", diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 1fe784455c..dd7274ac1f 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -1147,6 +1147,7 @@ def forward( position_ids=None, image_bound=None, tgt_sizes=None, + target_sizes=None, pixel_values_videos=None, image_grid_thw=None, video_grid_thw=None, @@ -1177,6 +1178,7 @@ def forward( past_key_values=past_key_values, image_bound=image_bound, tgt_sizes=tgt_sizes, + target_sizes=target_sizes, pixel_values_videos=pixel_values_videos, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, @@ -7363,11 +7365,256 @@ def generate(self, *args, **kwargs): _OVQwen3_5ForCausalLM.rot_pos_emb = Qwen3_5VisionModel.rot_pos_emb +class _OVMiniCPMV4_6ForCausalLM(OVModelForVisualCausalLM): + """OpenVINO runtime for MiniCPM-V-4.6. + + The exported ``vision_embeddings`` graph performs the entire image feature + extraction (NaViT SigLIP encoder, ViT window-attention merger, downsample + merger) but expects all data-dependent index / mask tensors to be provided. + This class reproduces the upstream ``MiniCPMV4_6`` NaViT packing logic in + Python to build those tensors, runs the vision graph, and inserts the + resulting image features into the text embeddings with a ``masked_scatter`` + on ``image_token_id`` (matching ``MiniCPMV4_6Model.forward``). The + ``qwen3_5_text`` backbone uses standard 1D RoPE position ids, so no 3D + position handling is needed. + """ + + def __init__( + self, + language_model: ov.Model, + text_embeddings: ov.Model, + vision_embeddings: ov.Model, + config: PretrainedConfig = None, + device: str = "CPU", + dynamic_shapes: bool = None, + ov_config: Optional[Dict[str, str]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + quantization_config: Union[OVWeightQuantizationConfig, Dict] = None, + **kwargs, + ): + if is_transformers_version("<", "5.7.0"): + raise Exception("MiniCPM-V-4.6 requires transformers >= 5.7.0; earlier versions are not supported.") + super().__init__( + language_model=language_model, + text_embeddings=text_embeddings, + vision_embeddings=vision_embeddings, + config=config, + device=device, + dynamic_shapes=dynamic_shapes, + ov_config=ov_config, + model_save_dir=model_save_dir, + quantization_config=quantization_config, + **kwargs, + ) + vision_config = self.config.vision_config + self.num_patches_per_side = vision_config.image_size // vision_config.patch_size + self.insert_layer_id = vision_config.insert_layer_id + self.window_kernel_size = tuple(getattr(vision_config, "window_kernel_size", (2, 2))) + self.merge_kernel_size = tuple(self.config.merge_kernel_size) + self.downsample_mode = getattr(self.config, "downsample_mode", "16x") + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + inputs_embeds=None, + pixel_values=None, + attention_mask=None, + **kwargs, + ): + target_sizes = kwargs.pop("target_sizes", None) + model_inputs = super().prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + attention_mask=attention_mask, + **kwargs, + ) + # Only feed image tensors during prefill. + if past_key_values is not None: + model_inputs["pixel_values"] = None + model_inputs["target_sizes"] = None + else: + model_inputs["target_sizes"] = target_sizes + return model_inputs + + @staticmethod + def _block_diagonal_mask(seqlens): + total = int(sum(seqlens)) + mask = torch.full((1, total, total), float("-inf")) + offset = 0 + for length in seqlens: + length = int(length) + mask[0, offset : offset + length, offset : offset + length] = 0.0 + offset += length + return mask + + def _patch_position_ids(self, target_sizes): + # Reproduces the upstream NaViT nearest-neighbour patch position ids used by + # ``MiniCPMV4_6VisionEmbeddings.forward`` via ``get_vision_nearest_position_ids``: + # https://github.com/huggingface/transformers/blob/main/src/transformers/vision_utils.py + # (see ``MiniCPMV4_6VisionEmbeddings`` in + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py). + num_side = self.num_patches_per_side + boundaries = torch.arange(1 / num_side, 1.0, 1 / num_side) + pos_ids = [] + for target_size in target_sizes: + nb_h, nb_w = int(target_size[0]), int(target_size[1]) + fractional_h = torch.arange(0, 1 - 1e-6, 1 / nb_h) + fractional_w = torch.arange(0, 1 - 1e-6, 1 / nb_w) + bucket_h = torch.bucketize(fractional_h, boundaries, right=True) + bucket_w = torch.bucketize(fractional_w, boundaries, right=True) + pos_ids.append((bucket_h[:, None] * num_side + bucket_w).flatten()) + return torch.cat(pos_ids) + + def _window_index(self, target_sizes): + # Reproduces the upstream window reordering + ``cu_seqlens`` computed by + # ``MiniCPMV4_6ViTWindowAttentionMerger.get_window_index`` (which calls + # ``get_vision_window_index``): + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py + # https://github.com/huggingface/transformers/blob/main/src/transformers/vision_utils.py + window_h, window_w = self.window_kernel_size + window_index_list = [] + cu_seqlens = [0] + token_offset = 0 + for height, width in target_sizes.tolist(): + height, width = int(height), int(width) + index = torch.arange(height * width).reshape(height, width) + num_windows_h = height // window_h + num_windows_w = width // window_w + num_windows = num_windows_h * num_windows_w + index = index.reshape(num_windows_h, window_h, num_windows_w, window_w) + index = index.permute(0, 2, 1, 3).reshape(num_windows, window_h * window_w) + window_index_list.append(index.reshape(-1) + token_offset) + cu_this = torch.arange(1, num_windows + 1) * (window_h * window_w) + cu_seqlens[-1] + cu_seqlens.extend(cu_this.tolist()) + token_offset += height * width + window_index = torch.cat(window_index_list) + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) + return window_index, cu_seqlens + + def get_vision_embeddings(self, pixel_values, input_ids=None, **kwargs): + # Python-side reproduction of the upstream NaViT vision pipeline + # (``MiniCPMV4_6Model.get_image_features`` -> + # ``MiniCPMV4_6VisionModel.forward`` -> ``MiniCPMV4_6ViTWindowAttentionMerger.forward`` + # -> ``MiniCPMV4_6Merger.forward``). The window-merge / spatial-merge gather + # indices below mirror the reshape+permute merging done in those modules: + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py + if input_ids is not None and input_ids.shape[1] == 1: + return None + target_sizes = kwargs.get("target_sizes") + if target_sizes is None: + return None + if not isinstance(pixel_values, torch.Tensor): + pixel_values = torch.as_tensor(pixel_values) + if not isinstance(target_sizes, torch.Tensor): + target_sizes = torch.as_tensor(target_sizes) + # NaViT packing => encode only the first (packed) batch element. + pixel_values = pixel_values[:1].to(torch.float32) + + window_h, window_w = self.window_kernel_size + merge_h, merge_w = self.merge_kernel_size + + # ---- Python precompute of all index / mask tensors ---- + pos_ids = self._patch_position_ids(target_sizes) + encoder_seqlens = [int(h * w) for h, w in target_sizes.tolist()] + encoder_mask = self._block_diagonal_mask(encoder_seqlens) + + window_index, window_cu = self._window_index(target_sizes) + reverse_window_index = torch.argsort(window_index) + window_seqlens = (window_cu[1:] - window_cu[:-1]).tolist() + window_mask = self._block_diagonal_mask(window_seqlens) + + # spatial merge gather (per image, concatenated). Each window of + # ``window_h * window_w`` patches becomes contiguous, so the exported graph + # can collapse all tiles with a ``-1`` row dimension. + merge_gather = [] + offset = 0 + for height, width in target_sizes.tolist(): + height, width = int(height), int(width) + mh, mw = height // window_h, width // window_w + base = torch.arange(height * width).reshape(mh, window_h, mw, window_w) + base = base.permute(0, 2, 1, 3).reshape(-1) + offset + merge_gather.append(base) + offset += height * width + merge_gather = torch.cat(merge_gather) + + downsampled_sizes = target_sizes // 2 + downsampled_seqlens = [int(h * w) for h, w in downsampled_sizes.tolist()] + downsampled_mask = self._block_diagonal_mask(downsampled_seqlens) + + final_gather = [] + offset = 0 + for height, width in downsampled_sizes.tolist(): + height, width = int(height), int(width) + fh, fw = height // merge_h, width // merge_w + base = torch.arange(height * width).reshape(fh, merge_h, fw, merge_w) + base = base.permute(0, 2, 1, 3).reshape(-1) + offset + final_gather.append(base) + offset += height * width + final_gather = torch.cat(final_gather) + + image_features = self.vision_embeddings( + pixel_values=pixel_values.numpy(), + pos_ids=pos_ids.numpy(), + encoder_attention_mask=encoder_mask.numpy(), + downsampled_attention_mask=downsampled_mask.numpy(), + window_index=window_index.numpy(), + reverse_window_index=reverse_window_index.numpy(), + window_attention_mask=window_mask.numpy(), + merge_gather_index=merge_gather.numpy(), + final_gather_index=final_gather.numpy(), + )[0] + return torch.from_numpy(image_features) + + def merge_vision_text_embeddings( + self, vision_embeds, inputs_embeds, input_ids=None, attention_mask=None, position_ids=None, **kwargs + ): + # Reproduces the ``masked_scatter`` on ``image_token_id`` performed by + # ``MiniCPMV4_6Model.forward``: + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/minicpmv4_6/modeling_minicpmv4_6.py + inputs_embeds = torch.from_numpy(inputs_embeds) if isinstance(inputs_embeds, np.ndarray) else inputs_embeds + image_features = ( + torch.from_numpy(vision_embeds) if isinstance(vision_embeds, np.ndarray) else vision_embeds + ) + image_features = image_features.to(inputs_embeds.dtype) + image_token_id = self.config.image_token_id + mask = input_ids == image_token_id + mask = mask.unsqueeze(-1).expand_as(inputs_embeds) + inputs_embeds = inputs_embeds.masked_scatter(mask, image_features) + return inputs_embeds, attention_mask, position_ids + + @staticmethod + def preprocess_inputs( + text: str, + image: Optional["Image"] = None, + processor: Optional[AutoImageProcessor] = None, + tokenizer: Optional[PreTrainedTokenizer] = None, + config: Optional[PretrainedConfig] = None, + video: Optional["VideoInput"] = None, + audio: Optional[np.ndarray] = None, + ): + if processor is None: + raise ValueError("Processor is required.") + if audio is not None: + raise ValueError("Audio input is not supported") + conversation = [{"role": "user", "content": [{"type": "text", "text": text}]}] + if image is not None: + conversation[0]["content"].insert(0, {"type": "image"}) + if video is not None: + conversation[0]["content"].insert(0, {"type": "video"}) + text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True) + inputs = processor(images=image, text=text_prompt, videos=video, return_tensors="pt") + return inputs + + MODEL_TYPE_TO_CLS_MAPPING = { "llava": _OVLlavaForCausalLM, "llava_next": _OVLlavaNextForCausalLM, "llava_next_video": _OVLlavaNextVideoForCausalLM, "minicpmv": _OVMiniCPMVForCausalLM, + "minicpmv4_6": _OVMiniCPMV4_6ForCausalLM, "llava-qwen2": _OVNanoLlavaForCausalLM, "maira2": _OVMaira2ForCausalLM, "phi3_v": _OVPhi3VisionForCausalLM, diff --git a/tests/openvino/test_export.py b/tests/openvino/test_export.py index d9928449a3..8c57162472 100644 --- a/tests/openvino/test_export.py +++ b/tests/openvino/test_export.py @@ -118,6 +118,7 @@ class ExportModelTest(unittest.TestCase): "gemma4_moe": OVModelForVisualCausalLM, "qwen3_5": OVModelForVisualCausalLM, "qwen3_5_moe": OVModelForVisualCausalLM, + "minicpmv4_6": OVModelForVisualCausalLM, "gemma4_unified": OVModelForVisualCausalLM, "gemma3n": OVModelForVisualCausalLM, "flux.2-klein": OVFlux2KleinPipeline, diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 371b729b5e..f9c7ab8b26 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -600,6 +600,7 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): "qwen3_5", "qwen3_5_moe", "qwen3_omni_moe", + "minicpmv4_6", ] SUPPORT_VIDEO = ["llava_next_video", "qwen2_vl", "qwen2_5_vl", "qwen3_vl", "videochat_flash_qwen"] SUPPORT_AUDIO = ["qwen3_omni_moe"] @@ -661,6 +662,7 @@ def get_transformer_model_class(self, model_arch): "qwen3_5", "qwen3_5_moe", "gemma4_unified", + "minicpmv4_6", ]: from transformers import AutoModelForImageTextToText diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 74418694cc..f5ef348b74 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -30,9 +30,81 @@ from optimum.intel.utils.import_utils import is_transformers_version +def _create_tiny_minicpmv4_6_model(): + """Generate a tiny random MiniCPM-V-4.6 model for testing and return its local path. + + Preserves the real ``minicpmv4_6`` architecture: a NaViT-packed + ``minicpmv4_6_vision`` encoder with a ViT window-attention merger + downsample + merger, and a ``qwen3_5_text`` hybrid (linear + full attention) language backbone. + Scale parameters (hidden sizes, layer counts) are reduced while the (tied) + embedding table keeps the real vocabulary so the MiniCPM-V tokenizer chat-template + token ids stay in range. Cached on disk so repeated calls are cheap. + """ + import torch + + output_dir = Path(tempfile.gettempdir()) / "optimum_intel_tiny_random_minicpmv4_6" + marker = output_dir / "tiny_minicpmv4_6_v1" + if marker.exists() and (output_dir / "config.json").exists(): + return str(output_dir) + + from transformers import AutoConfig, AutoProcessor, MiniCPMV4_6ForConditionalGeneration + + torch.manual_seed(42) + base_id = "openbmb/MiniCPM-V-4.6" + config = AutoConfig.from_pretrained(base_id) + + tc = config.text_config + tc.num_hidden_layers = 4 + tc.layer_types = ["linear_attention", "linear_attention", "linear_attention", "full_attention"] + tc.hidden_size = 64 + tc.intermediate_size = 128 + tc.num_attention_heads = 4 + tc.num_key_value_heads = 2 + tc.head_dim = 64 + tc.linear_key_head_dim = 32 + tc.linear_value_head_dim = 32 + tc.linear_num_key_heads = 4 + tc.linear_num_value_heads = 4 + tc.linear_conv_kernel_dim = 4 + tc.max_position_embeddings = 4096 + + vc = config.vision_config + vc.hidden_size = 128 + vc.intermediate_size = 256 + vc.num_hidden_layers = 4 + vc.num_attention_heads = 4 + vc.patch_size = 14 + vc.image_size = 980 + + config.insert_layer_id = 1 + config.vocab_size = tc.vocab_size + config.tie_word_embeddings = True + tc.image_token_id = config.image_token_id + tc.video_token_id = config.video_token_id + config.dtype = "float32" + config.torch_dtype = "float32" + for sub in (tc, vc): + sub.dtype = "float32" + sub.torch_dtype = "float32" + + model = MiniCPMV4_6ForConditionalGeneration(config).to(torch.float32).eval() + with torch.no_grad(): + for name, param in model.named_parameters(): + if "language_model" in name and param.dim() == 2 and ( + "proj" in name or "mlp" in name or "fc" in name + ): + param.normal_(mean=0.0, std=0.12) + model.get_input_embeddings().weight.normal_(mean=0.0, std=0.18) + + output_dir.mkdir(parents=True, exist_ok=True) + model.save_pretrained(output_dir, safe_serialization=True) + AutoProcessor.from_pretrained(base_id).save_pretrained(output_dir) + marker.write_text("tiny_minicpmv4_6_v1") + return str(output_dir) + + def _create_tiny_kokoro_model(): """Generate a tiny random Kokoro TTS model for testing and return its local path. - Falls back to the original Hub id if the `kokoro` package is not installed. Result is cached on disk under the system temp dir, so subsequent calls are cheap. """ @@ -43,8 +115,13 @@ def _create_tiny_kokoro_model(): if config_file.exists() and weights_file.exists() and voice_file.exists(): return str(output_dir) - from kokoro.istftnet import Decoder - from kokoro.modules import CustomAlbert, ProsodyPredictor, TextEncoder + try: + from kokoro.istftnet import Decoder + from kokoro.modules import CustomAlbert, ProsodyPredictor, TextEncoder + except ImportError: + # Honor the documented fallback: if the optional `kokoro` package is not + # installed, use the original Hub id instead of breaking test collection. + return "hexgrad/Kokoro-82M" from transformers import AlbertConfig output_dir.mkdir(parents=True, exist_ok=True) @@ -262,6 +339,9 @@ def _create_tiny_kokoro_model(): "minicpm": "optimum-intel-internal-testing/tiny-random-minicpm", "minicpm3": "optimum-intel-internal-testing/tiny-random-minicpm3", "minicpmv": "optimum-intel-internal-testing/tiny-random-minicpmv-2_6", + "minicpmv4_6": _create_tiny_minicpmv4_6_model() + if is_transformers_version(">=", "5.7.0") + else "openbmb/MiniCPM-V-4.6", "minicpmo": "optimum-intel-internal-testing/tiny-random-MiniCPM-o-2_6", "mistral": "optimum-intel-internal-testing/tiny-random-mistral", "mistral-nemo": "optimum-intel-internal-testing/tiny-random-mistral-nemo", @@ -490,6 +570,11 @@ def _resolve_cached_model_paths(model_names: dict) -> dict: "vision_embeddings_model": 26, "resampler_model": 6, }, + "minicpmv4_6": { + "lm_model": 70, + "text_embeddings_model": 1, + "vision_embeddings_model": 34, + }, "llava_next_video": { "lm_model": 30, "text_embeddings_model": 1,