From 3229b1a90be71a858fd7b0c71acb9e6c815edae2 Mon Sep 17 00:00:00 2001 From: Omega-Intel Date: Wed, 29 Jul 2026 20:49:39 +0200 Subject: [PATCH] Fix VLM Eagle3 draft model export/load regression (qwen3_vl_eagle3) Two distinct bugs affected VLM-flavored Eagle3 speculative-decoding draft models (e.g. AngelSlim/Qwen3-VL-4B-Instruct_eagle3), reported in openvinotoolkit/omega#78: 1. Export (`optimum-cli export openvino`, any weight format) failed with `KeyError: 'input_ids'` in `OpenVINOConfigWithPast.generate_dummy_inputs`. VLM Eagle3 configs replace `input_ids` with `inputs_embeds` in dummy inputs, but the attention-mask padding branch only looked up `input_ids` and only triggered for `task == "text-generation"`, missing `image-text-to-text` (the task under which `qwen3_vl_eagle3` registers). Restored the `.get()` fallback and task condition that existed prior to the base.py export-config refactor. 2. Loading an exported VLM Eagle3 model via `OVModelForVisualCausalLM` failed with a cryptic `KeyError: 'llama'` in `_from_pretrained`, because these checkpoints self-report `model_type="llama"` (not a key in `MODEL_TYPE_TO_CLS_MAPPING`) while carrying VLM-oriented `modal_type`/`target_model_type` fields. These models are standalone draft causal LMs and must be loaded with `OVModelForCausalLM` instead. Added a pre-check that raises a clear, actionable `ValueError` pointing at the correct class. Added `test_exporters_cli_eagle3_vlm_quantization` covering fp16/int8/ int4 export + load for `qwen3_vl_eagle3`, using the existing tiny CI fixture, and asserting the improved error message from bug #2. --- optimum/exporters/openvino/base.py | 6 +++-- .../openvino/modeling_visual_language.py | 13 ++++++++++ tests/openvino/test_exporters_cli.py | 26 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/optimum/exporters/openvino/base.py b/optimum/exporters/openvino/base.py index fe3ec819ae..86e9b258b6 100644 --- a/optimum/exporters/openvino/base.py +++ b/optimum/exporters/openvino/base.py @@ -352,9 +352,11 @@ def generate_dummy_inputs(self, framework: str = "pt", **kwargs): and self.PAD_ATTENTION_MASK_TO_PAST and self.use_cache_branch is not False and "attention_mask" in dummy_inputs - and self.task == "text-generation" + and self.task in ("text-generation", "image-text-to-text") ): - seq_len = dummy_inputs["input_ids"].shape[1] + # VLM Eagle3 uses inputs_embeds instead of input_ids + main_input = dummy_inputs.get("input_ids", dummy_inputs.get("inputs_embeds")) + seq_len = main_input.shape[1] past_seq_len = dummy_inputs["past_key_values"][0][1].shape[-2] dummy_inputs["attention_mask"] = DummyInputGenerator.pad_input_on_dim( dummy_inputs["attention_mask"], desired_length=past_seq_len + seq_len, dim=1 diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 1fe784455c..9e335e06c7 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -895,6 +895,19 @@ def _from_pretrained( trust_remote_code (`bool`, *optional*, defaults to `False`): Whether to trust remote code when loading model tokenizer/processor during quantization. """ + if config.model_type not in MODEL_TYPE_TO_CLS_MAPPING: + archs = getattr(config, "architectures", None) or [] + if archs and "eagle3" in archs[0].lower(): + raise ValueError( + f"Model with architecture '{archs[0]}' (model_type='{config.model_type}') is a standalone " + "Eagle3 speculative-decoding draft model, not a multi-component VLM, even though its " + "config declares a VLM-oriented `modal_type`/`target_model_type`. Please load it with " + "`OVModelForCausalLM` instead of `OVModelForVisualCausalLM`." + ) + raise ValueError( + f"Unsupported model_type '{config.model_type}' for `OVModelForVisualCausalLM`. Supported " + f"model types are: {sorted(MODEL_TYPE_TO_CLS_MAPPING)}." + ) model_cls = MODEL_TYPE_TO_CLS_MAPPING[config.model_type] model_file_names = model_cls._all_ov_model_paths.copy() for k in tuple(model_file_names): diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index c57789a4c5..d082354750 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -1114,6 +1114,32 @@ def test_exporters_cli_int8(self, task: str, model_type: str): del expected_int8["decoder_with_past"] check_compression_state_per_model(self, model.ov_models, expected_int8) + @parameterized.expand(["fp16", "int8", "int4"]) + def test_exporters_cli_eagle3_vlm_quantization(self, weight_format: str): + # Regression test: exporting the VLM-flavored Eagle3 draft model (e.g. AngelSlim's + # Qwen3-VL eagle3 checkpoints) with any weight format used to fail with + # `KeyError: 'input_ids'` in `generate_dummy_inputs` because `eagle3_vlm` configs + # replace `input_ids` with `inputs_embeds` (see model_configs.py LlamaOpenVINOConfig). + model_type = "qwen3_vl_eagle3" + task = "text-generation-with-past" + with TemporaryDirectory() as tmpdir: + add_ops = "--group-size 16" if weight_format == "int4" else "" + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} " + f"--trust-remote-code --weight-format {weight_format} {add_ops} {tmpdir}", + shell=True, + check=True, + ) + # Must be loaded with OVModelForCausalLM: it is a standalone draft causal LM, + # not a multi-component VLM, even though its config carries VLM-oriented fields. + model = OVModelForCausalLM.from_pretrained(tmpdir, use_cache=True) + self.assertTrue(model.stateful) + + # Loading the same export with OVModelForVisualCausalLM must raise a clear, + # actionable error instead of a bare `KeyError: 'llama'`. + with self.assertRaisesRegex(ValueError, "OVModelForCausalLM"): + OVModelForVisualCausalLM.from_pretrained(tmpdir, use_cache=True) + @parameterized.expand(SUPPORTED_SD_HYBRID_ARCHITECTURES) def test_exporters_cli_hybrid_quantization( self, model_type: str, expected_fake_nodes: int, expected_int8_nodes: int