diff --git a/site/docs/supported-models/_components/llm-models-table/models.ts b/site/docs/supported-models/_components/llm-models-table/models.ts index 1a542e2ca4..01113a4cc4 100644 --- a/site/docs/supported-models/_components/llm-models-table/models.ts +++ b/site/docs/supported-models/_components/llm-models-table/models.ts @@ -229,6 +229,21 @@ export const LLM_MODELS: LLMModelType[] = [ }, ], }, + { + architecture: 'FalconH1ForCausalLM', + models: [ + { + name: 'Falcon-H1', + links: [ + 'https://huggingface.co/tiiuae/Falcon-H1-0.5B-Instruct', + 'https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base', + 'https://huggingface.co/tiiuae/Falcon-H1-1.5B-Instruct', + 'https://huggingface.co/tiiuae/Falcon-H1-3B-Instruct', + 'https://huggingface.co/tiiuae/Falcon-H1-7B-Instruct', + ], + }, + ], + }, { architecture: 'GemmaForCausalLM', models: [ diff --git a/src/cpp/src/llm/pipeline.cpp b/src/cpp/src/llm/pipeline.cpp index 4f9ba7dd5a..ccbbe0d199 100644 --- a/src/cpp/src/llm/pipeline.cpp +++ b/src/cpp/src/llm/pipeline.cpp @@ -259,7 +259,20 @@ ov::genai::LLMPipeline::LLMPipeline( } else if (utils::explicitly_requires_paged_attention(user_properties)) { // If CB is invoked explicitly, create CB adapter as is and re-throw in case if internal issues auto [device_properties, scheduler_config] = utils::extract_scheduler_config(properties, utils::get_latency_oriented_scheduler_config()); - m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config, models_path); + try { + m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config, models_path); + } catch (const ov::Exception& exception) { + // Hybrid Mamba/attention models (e.g. Falcon-H1) cannot be converted to PagedAttention: their recurrent + // state leaves a dangling beam_idx parameter. PagedAttention cannot be honored for such models, so fall + // back to the stateful backend instead of surfacing the low-level failure. + if (!utils::is_paged_attention_model_construction_failure(exception)) { + throw; + } + log_paged_attention_fallback(exception); + // Drop any scheduler_config so the stateful backend does not forward a PagedAttention-only property. + properties.erase(ov::genai::scheduler_config.name()); + model = utils::read_model(models_path, properties); + } } else if (attention_backend == PA_BACKEND) { try { // we need use CB only for x86 and arm64, as for other architectures like risc-v we can create Paged Attention based model @@ -304,7 +317,20 @@ ov::genai::LLMPipeline::LLMPipeline( } else if (utils::explicitly_requires_paged_attention(user_properties)) { // If CB is invoked explicitly, create CB adapter as is and re-throw in case if internal issues auto [device_properties, scheduler_config] = utils::extract_scheduler_config(properties, utils::get_latency_oriented_scheduler_config()); - m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config, models_path); + try { + m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config, models_path); + } catch (const ov::Exception& exception) { + // Hybrid Mamba/attention models (e.g. Falcon-H1) cannot be converted to PagedAttention: their recurrent + // state leaves a dangling beam_idx parameter. PagedAttention cannot be honored for such models, so fall + // back to the stateful backend instead of surfacing the low-level failure. + if (!utils::is_paged_attention_model_construction_failure(exception)) { + throw; + } + log_paged_attention_fallback(exception); + // Drop any scheduler_config so the stateful backend does not forward a PagedAttention-only property. + properties.erase(ov::genai::scheduler_config.name()); + model = utils::read_model(models_path, properties); + } } else if (attention_backend == PA_BACKEND) { // try to call CB adapter one more time, but with safe guard to silent exception try { @@ -355,7 +381,20 @@ ov::genai::LLMPipeline::LLMPipeline( } else if (utils::explicitly_requires_paged_attention(user_properties)) { // If CB is invoked explicitly, create CB adapter as is and re-throw in case if internal issues auto [device_properties, scheduler_config] = utils::extract_scheduler_config(properties, utils::get_latency_oriented_scheduler_config()); - m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config); + try { + m_pimpl = std::make_unique(model, tokenizer, scheduler_config, device, device_properties, generation_config); + } catch (const ov::Exception& exception) { + // Hybrid Mamba/attention models (e.g. Falcon-H1) cannot be converted to PagedAttention: their recurrent + // state leaves a dangling beam_idx parameter. PagedAttention cannot be honored for such models, so fall + // back to the stateful backend instead of surfacing the low-level failure. + if (!utils::is_paged_attention_model_construction_failure(exception)) { + throw; + } + log_paged_attention_fallback(exception); + // Drop any scheduler_config so the stateful backend does not forward a PagedAttention-only property. + properties.erase(ov::genai::scheduler_config.name()); + model = utils::singleton_core().read_model(model_str, weights_tensor); + } } else if (attention_backend == PA_BACKEND) { // try to call CB adapter one more time, but with safe guard to silent exception try { diff --git a/src/cpp/src/utils.cpp b/src/cpp/src/utils.cpp index 387455d549..88c181f871 100644 --- a/src/cpp/src/utils.cpp +++ b/src/cpp/src/utils.cpp @@ -934,6 +934,20 @@ std::pair extract_attention_backend(const ov::AnyMap& e return {properties, attention_backend}; }; +bool is_paged_attention_model_construction_failure(const ov::Exception& exception) { + const std::string message = exception.what(); + // The SDPAToPagedAttention transformation fails to convert hybrid Mamba/attention models (e.g. Falcon-H1): + // the recurrent convolution/SSM state keeps referencing the beam_idx parameter after the parameter itself is + // removed, which ov::Model validation reports as an undeclared/unregistered beam_idx parameter. Match this + // specific model-construction failure so only genuinely PagedAttention-incompatible models fall back to the + // stateful backend, while models that merely hit a transient runtime error still surface it. + const bool references_beam_idx = message.find("beam_idx") != std::string::npos; + const bool undeclared_parameter = + message.find("undeclared parameters") != std::string::npos || + message.find("unregistered_parameters") != std::string::npos; + return references_beam_idx && undeclared_parameter; +} + ExtensionList extract_extensions(ov::AnyMap& properties) { auto it = properties.find(EXTENSIONS_ARG_NAME); ExtensionList extensions = {}; diff --git a/src/cpp/src/utils.hpp b/src/cpp/src/utils.hpp index 89a64b352c..446a9a23b1 100644 --- a/src/cpp/src/utils.hpp +++ b/src/cpp/src/utils.hpp @@ -351,6 +351,21 @@ bool explicitly_requires_paged_attention(const ov::AnyMap& properties, bool is_n std::pair extract_attention_backend(const ov::AnyMap& external_properties, bool is_npu_requested = false); +/** + * @brief Detects whether an exception raised while constructing a PagedAttention/ContinuousBatching pipeline + * indicates that the model cannot be converted to PagedAttention at all. + * + * Hybrid architectures such as Falcon-H1 (Mamba/SSM mixed with attention) carry recurrent convolution/SSM state + * that the SDPAToPagedAttention transformation cannot rewrite. The recurrent state keeps referencing the model's + * beam_idx parameter, which the transformation removes, producing an "undeclared parameter beam_idx" model + * validation failure. Such models can only run with the stateful backend, so the pipeline should fall back to it + * instead of surfacing the low-level failure. + * + * @param exception The exception thrown during PagedAttention pipeline construction. + * @return true if the failure is a PagedAttention model-construction incompatibility that warrants a stateful fallback. + */ +bool is_paged_attention_model_construction_failure(const ov::Exception& exception); + /** * @brief Extracts the "extensions" key from the provided properties map and returns the corresponding * list of extensions. diff --git a/tests/python_tests/data/models.py b/tests/python_tests/data/models.py index 494069aad6..aa0dd28c97 100644 --- a/tests/python_tests/data/models.py +++ b/tests/python_tests/data/models.py @@ -27,7 +27,17 @@ def get_models_list() -> tuple[str, ...]: ) elif is_transformers_version(">=", "4.57"): # Restore after fix https://github.com/huggingface/optimum-intel/pull/1589 - LINEAR_ATTENTION_MODELS_LIST = ("optimum-intel-internal-testing/tiny-random-qwen3-next",) + LINEAR_ATTENTION_MODELS_LIST = ( + "optimum-intel-internal-testing/tiny-random-qwen3-next", + # Falcon-H1 (hybrid Mamba + attention) is enabled for GenAI: an explicit PagedAttention request now falls + # back to the stateful backend when the SDPAToPagedAttention transformation cannot rewrite the recurrent + # conv/ssm state (which otherwise fails with a dangling beam_idx parameter). Enable this entry once a + # non-degenerate tiny-random FalconH1 fixture is published under the optimum-intel-internal-testing + # namespace. The currently available public tiny-random FalconH1 checkpoints (e.g. tiny-random/falcon-h1, + # yujiepan/falcon-h1-tiny-random) use degenerate proportions (hidden_size=8, mamba_expand=32) that crash + # the Mamba naive kernel during generation and cannot serve as a reliable regression fixture. + # "optimum-intel-internal-testing/tiny-random-falcon-h1", + ) GGUF_MODEL_LIST = (