diff --git a/.github/workflows/test_openvino_preview_models.yml b/.github/workflows/test_openvino_preview_models.yml index 758a8167e6..d08c8ad2b0 100644 --- a/.github/workflows/test_openvino_preview_models.yml +++ b/.github/workflows/test_openvino_preview_models.yml @@ -70,4 +70,3 @@ jobs: run: | uv pip install "transformers>=5.10,<5.11" pytest tests/openvino/${{ matrix.test-pattern }} -k gemma4_unified --durations=0 - diff --git a/.github/workflows/test_openvino_slow.yml b/.github/workflows/test_openvino_slow.yml index 5302fc62e8..76758b51af 100644 --- a/.github/workflows/test_openvino_slow.yml +++ b/.github/workflows/test_openvino_slow.yml @@ -70,11 +70,6 @@ jobs: path: ~/.cache/huggingface/hub key: hf-models - - if: ${{ matrix.transformers-version != 'latest' && matrix.transformers-version != 'main' }} - name: Install specific dependencies and versions required for older transformers - run: | - uv pip install transformers==${{ matrix.transformers-version }} accelerate==0.* peft==0.13.* diffusers==0.32.* transformers_stream_generator - - if: ${{ matrix.transformers-version == 'main' }} name: Install transformers from repository run: | diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index b2c375048a..7c696a2bc8 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -1413,7 +1413,9 @@ def get_sd3_models_for_export(pipeline, exporter, int_dtype, float_dtype): text_encoder = getattr(pipeline, "text_encoder", None) if text_encoder is not None: text_encoder.config.output_hidden_states = True - text_encoder.text_model.config.output_hidden_states = True + # `CLIPTextTransformer` removed since transformers v5.6 + if hasattr(text_encoder, "text_model"): + text_encoder.text_model.config.output_hidden_states = True text_encoder_config_constructor = TasksManager.get_exporter_config_constructor( model=text_encoder, exporter=exporter, @@ -1475,7 +1477,9 @@ def get_sd3_models_for_export(pipeline, exporter, int_dtype, float_dtype): text_encoder_2 = getattr(pipeline, "text_encoder_2", None) if text_encoder_2 is not None: text_encoder_2.config.output_hidden_states = True - text_encoder_2.text_model.config.output_hidden_states = True + # `CLIPTextTransformer` removed since transformers v5.6 + if hasattr(text_encoder_2, "text_model"): + text_encoder_2.text_model.config.output_hidden_states = True export_config_constructor = TasksManager.get_exporter_config_constructor( model=text_encoder_2, exporter=exporter, diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 7b067aeb17..69af94d1d9 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -4959,29 +4959,33 @@ class GraniteMoEModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - for layer in self._model.model.layers: - block_sparse_moe = layer.block_sparse_moe - block_sparse_moe.router._orig_forward = block_sparse_moe.router.forward - block_sparse_moe.router.forward = types.MethodType( - _granite_moe_topk_gating_forward, block_sparse_moe.router - ) - block_sparse_moe.input_linear._orig_forward = block_sparse_moe.input_linear.forward - block_sparse_moe.input_linear.forward = types.MethodType( - _granite_moe_parallel_experts_forward, block_sparse_moe.input_linear - ) - block_sparse_moe.output_linear._orig_forward = block_sparse_moe.output_linear.forward - block_sparse_moe.output_linear.forward = types.MethodType( - _granite_moe_parallel_experts_forward, block_sparse_moe.output_linear - ) + if is_transformers_version("<", "5.13"): + for layer in self._model.model.layers: + block_sparse_moe = layer.block_sparse_moe + block_sparse_moe.router._orig_forward = block_sparse_moe.router.forward + block_sparse_moe.router.forward = types.MethodType( + _granite_moe_topk_gating_forward, block_sparse_moe.router + ) + block_sparse_moe.input_linear._orig_forward = block_sparse_moe.input_linear.forward + block_sparse_moe.input_linear.forward = types.MethodType( + _granite_moe_parallel_experts_forward, block_sparse_moe.input_linear + ) + block_sparse_moe.output_linear._orig_forward = block_sparse_moe.output_linear.forward + block_sparse_moe.output_linear.forward = types.MethodType( + _granite_moe_parallel_experts_forward, block_sparse_moe.output_linear + ) + else: + register_ov_batched_mm(self) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - for layer in self._model.model.layers: - block_sparse_moe = layer.block_sparse_moe - block_sparse_moe.router.forward = block_sparse_moe.router._orig_forward - block_sparse_moe.input_linear.forward = block_sparse_moe.input_linear._orig_forward - block_sparse_moe.output_linear.forward = block_sparse_moe.output_linear._orig_forward + if is_transformers_version("<", "5.13"): + for layer in self._model.model.layers: + block_sparse_moe = layer.block_sparse_moe + block_sparse_moe.router.forward = block_sparse_moe.router._orig_forward + block_sparse_moe.input_linear.forward = block_sparse_moe.input_linear._orig_forward + block_sparse_moe.output_linear.forward = block_sparse_moe.output_linear._orig_forward class OVSeq2SeqModelPatcher(ModelPatcher): @@ -5171,12 +5175,6 @@ def forward_with_precomputed_mask(*args, **kwargs): self.orig_forward = forward_with_precomputed_mask - def __enter__(self): - super().__enter__() - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - # Forward method of the language model of Gemma3n, needs to be patched to pass 'per_layer_inputs', # as original code fails to create per_layer_inputs without the providing of input_ids, @@ -5259,7 +5257,7 @@ def gemma3n_language_model_forward( return outputs -# Creates a dict of causal masks with bidirectional attention for vision tokens +# Creates a dict of causal masks with bidirectional attention for vision tokens, # on sliding_attention layers, matching the behavior of transformers # create_causal_mask_mapping when use_bidirectional_attention == "vision". # Needs to be patched to pass proper 'sliding_mask' for prefill stage. @@ -5311,7 +5309,9 @@ def _create_gemma4_bidirectional_mask_dict(attention_mask_2d, mm_token_type_ids, same_group = (query_groups.unsqueeze(2) == key_groups.unsqueeze(1)) & (key_groups.unsqueeze(1) >= 0) same_group = same_group.unsqueeze(1) # [batch, 1, seq_len, total_len] - # Undo masking for same-group vision tokens in sliding mask + # Un-mask same-group vision tokens in both masks (bidirectional attention within an image). + if is_transformers_version(">=", "5.9") and is_transformers_version("<", "5.13"): + full_mask = full_mask.masked_fill(same_group, 0.0) sliding_mask = sliding_mask.masked_fill(same_group, 0.0) return { @@ -5519,6 +5519,10 @@ def gemma4_text_attention_forward( ) -> tuple: from transformers.models.gemma4.modeling_gemma4 import apply_rotary_pos_emb as apply_rotary_pos_emb_gemma4 + # since transformers >= v5.6 (PR #45788) `shared_kv_states` dict and is passed and `kv_shared_layer_index` removed + shared_kv_states = kwargs.pop("shared_kv_states", None) + legacy_shared_kv_states = hasattr(self, "kv_shared_layer_index") + input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.head_dim) @@ -5529,8 +5533,12 @@ def gemma4_text_attention_forward( query_states = apply_rotary_pos_emb_gemma4(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - if self.is_kv_shared_layer and past_key_values is not None: - key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + if self.is_kv_shared_layer and (not legacy_shared_kv_states or past_key_values is not None): + if legacy_shared_kv_states: + key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + else: + key_states, value_states = shared_kv_states[self.layer_type] + key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) else: @@ -5545,18 +5553,26 @@ def gemma4_text_attention_forward( value_states = value_states.transpose(1, 2) if past_key_values is not None: - cache_kwargs = { - "sin": sin, - "cos": cos, - "cache_position": cache_position, - "sliding_window": self.sliding_window, - } - if not self.is_kv_shared_layer: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) - if self.store_full_length_kv: - if not hasattr(past_key_values, "shared_layers"): - past_key_values.shared_layers = {} - past_key_values.shared_layers[self.layer_idx] = key_states, value_states + if legacy_shared_kv_states: + cache_kwargs = { + "sin": sin, + "cos": cos, + "cache_position": cache_position, + "sliding_window": self.sliding_window, + } + if not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + if self.store_full_length_kv: + if not hasattr(past_key_values, "shared_layers"): + past_key_values.shared_layers = {} + past_key_values.shared_layers[self.layer_idx] = key_states, value_states + else: + if not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + if self.store_full_length_kv and shared_kv_states is not None: + shared_kv_states[self.layer_type] = key_states, value_states attention_interface = gemma4_eager_attention_forward_patched @@ -7983,7 +7999,7 @@ def patch_sparse_moe(sparse_moe_layer): ) for idx, layer in enumerate(self._model.model.layers): - if getattr(layer, "block_sparse_moe", None) is not None: + if getattr(layer, "block_sparse_moe", None) is not None and is_transformers_version("<", "5.13"): patch_sparse_moe(layer.block_sparse_moe) if self.real_config._config.layers_block_type[idx] == "mamba": mamba_layer = layer.mamba @@ -7992,6 +8008,9 @@ def patch_sparse_moe(sparse_moe_layer): mamba_layer._orig_forward = mamba_layer.forward mamba_layer.forward = types.MethodType(zamba2_mamba_mixer, mamba_layer) + if is_transformers_version(">=", "5.13"): + register_ov_batched_mm(self) + def __exit__(self, exc_type, exc_value, traceback): def unpatch_sparse_moe(sparse_moe_layer): sparse_moe_layer.router.forward = sparse_moe_layer.router._orig_forward @@ -8005,7 +8024,7 @@ def unpatch_sparse_moe(sparse_moe_layer): self._model.model._update_causal_mask = self._model.model._orig_update_causal_mask for idx, layer in enumerate(self._model.model.layers): - if getattr(layer, "block_sparse_moe", None) is not None: + if getattr(layer, "block_sparse_moe", None) is not None and is_transformers_version("<", "5.13"): unpatch_sparse_moe(layer.block_sparse_moe) if self.real_config._config.layers_block_type[idx] == "mamba": mamba_layer = layer.mamba diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 882e123c77..57bf95c22d 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -1044,6 +1044,12 @@ def _incompatible_inputs_warning(self, inputs: Dict): return None + def get_experts_implementation(self): + return {} + + def set_experts_implementation(self, experts_implementation): + return + class OVModelPart(OVModelHostMixin): def __init__( diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 1fe784455c..73f7b20793 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -3160,9 +3160,12 @@ def rot_pos_emb(self, grid_thw): wpos_ids = wpos_ids.flatten() pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self._rotary_pos_emb(max_grid_size) - rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + if is_transformers_version(">=", "5.9"): + rotary_pos_emb = self._rotary_pos_emb(pos_ids) + else: + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self._rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb def get_multimodal_embeddings( @@ -4286,9 +4289,13 @@ def rot_pos_emb(self, grid_thw): ) pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) - rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + if is_transformers_version(">=", "5.9"): + rotary_pos_emb = self.rotary_pos_emb(pos_ids) + else: + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb def get_vision_embeddings(self, pixel_values, grid_thw, **kwargs): diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 54bc58d2ce..5c6dab23db 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -68,6 +68,9 @@ TEXTUAL_INVERSION_EMBEDDING_KEYS = [ "self.text_model.embeddings.token_embedding.weight", "self.model.text_model.embeddings.token_embedding.weight", + # `CLIPTextTransformer` was removed since transformers v5.6 + "self.embeddings.token_embedding.weight", + "self.model.embeddings.token_embedding.weight", ] OV_TO_NP_TYPE = { diff --git a/setup.py b/setup.py index 03ad50d507..815719a5a0 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum~=2.3.0", - "transformers>=4.51,<5.6", + "transformers>=4.51,<5.15", "setuptools", "huggingface-hub>=0.23.2,<1.22", "nncf>=3.3.0", diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 196d22a6a7..5c3fec1c03 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -64,6 +64,7 @@ SEED, TENSOR_ALIAS_TO_TYPE, TEST_IMAGE_URL, + is_model_type_transformers_compatible, ) from optimum.intel import ( @@ -1041,7 +1042,7 @@ class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase): "sentence-transformers-bert", "qwen3", ) - if is_transformers_version("<", "5.4") or is_transformers_version(">=", "5.6"): + if is_model_type_transformers_compatible("qwen3_vl"): SUPPORTED_ARCHITECTURES += ("qwen3_vl_embedding",) @parameterized.expand(SUPPORTED_ARCHITECTURES) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index e01080264f..e7d0ef3c25 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -1296,7 +1296,11 @@ class OVWeightCompressionTest(unittest.TestCase): { "unet": {"names": ["__module.time_embedding.linear_1/aten::linear/MatMul"]}, "text_encoder": { - "names": ["__module.text_model.encoder.layers.0.self_attn.q_proj/aten::linear/MatMul"] + "names": [ + "__module.text_model.encoder.layers.0.self_attn.q_proj/aten::linear/MatMul" + if is_transformers_version("<", "5.6") + else "__module.encoder.layers.0.self_attn.q_proj/aten::linear/MatMul" + ] }, }, ), diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index cd49b8b426..b318036b7c 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -612,13 +612,13 @@ def _resolve_cached_model_paths(model_names: dict) -> dict: "gemma4": { "lm_model": 54, "text_embeddings_model": 1, - "vision_embeddings_model": 10, + "vision_embeddings_model": 10 if is_transformers_version("<", "5.10") else 11, "text_embeddings_per_layer_model": 1, }, "gemma4_moe": { "lm_model": 48, "text_embeddings_model": 1, - "vision_embeddings_model": 10, + "vision_embeddings_model": 10 if is_transformers_version("<", "5.10") else 11, "text_embeddings_per_layer_model": 0, }, "smollm3": {"model": 30},