[OpenVINO] Add image-to-video support for LTX2

docs/source/openvino/inference.mdx

@@ -82,8 +82,10 @@ As shown in the table below, each task is associated with a class enabling to au
 | `OVStableDiffusionXLImg2ImgPipeline` | `image-to-image`                     |
 | `OVLatentConsistencyModelPipeline`   | `text-to-image`                      |
 | `OVLTXPipeline`                      | `text-to-video`                      |
+| `OVLTX2Pipeline`                     | `text-to-video`                      |
 | `OVPipelineForText2Video`            | `text-to-video`                      |
 | `OVLTXImageToVideoPipeline`          | `image-to-video`                     |
+| `OVLTX2ImageToVideoPipeline`         | `image-to-video`                     |
 | `OVPipelineForImage2Video`           | `image-to-video`                     |
 
 See the [reference documentation](reference) for more information about parameters, and examples for different tasks.

optimum/exporters/openvino/convert.py

@@ -1256,6 +1256,22 @@ def get_ltx2_video_models_for_export(pipeline, exporter, int_dtype, float_dtype)
     vae_decoder_export_config.runtime_options = {"ACTIVATIONS_SCALE_FACTOR": "8.0"}
     models_for_export["vae_decoder"] = (vae_decoder, vae_decoder_export_config)
 
+    # VAE Encoder (needed for image-to-video conditioning; harmless for text-to-video, which won't load it)
+    vae_encoder = copy.deepcopy(pipeline.vae)
+    vae_encoder.forward = lambda sample: {"latent_parameters": vae_encoder.encode(x=sample)["latent_dist"].parameters}
+    vae_encoder_config_constructor = TasksManager.get_exporter_config_constructor(
+        model=vae_encoder,
+        exporter=exporter,
+        library_name="diffusers",
+        task="semantic-segmentation",
+        model_type="ltx2-vae-encoder",
+    )
+    vae_encoder_export_config = vae_encoder_config_constructor(
+        vae_encoder.config, int_dtype=int_dtype, float_dtype=float_dtype
+    )
+    vae_encoder_export_config.runtime_options = {"ACTIVATIONS_SCALE_FACTOR": "8.0"}
+    models_for_export["vae_encoder"] = (vae_encoder, vae_encoder_export_config)
+
     # Audio VAE decoder and vocoder
     if hasattr(pipeline, "audio_vae") and pipeline.audio_vae is not None:
         # Audio VAE decoder

optimum/exporters/openvino/input_generators.py

@@ -865,9 +865,16 @@ def __init__(
     ):
         super().__init__(task, normalized_config, batch_size, num_channels, width, height, **kwargs)
         self.num_frames = num_frames
+        # Pixel-space properties used to shape the VAE encoder input (`sample`).
+        self.vae_in_channels = getattr(normalized_config.config, "in_channels", 3)
+        self.spatial_compression_ratio = getattr(normalized_config.config, "spatial_compression_ratio", 32)
 
     def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
-        if input_name in ["sample", "latent_sample"]:
+        if input_name == "sample":
+            # Pixel-space input with a single conditioning frame; num_frames=1 satisfies the temporal patchify.
+            spatial = self.spatial_compression_ratio
+            return self.random_float_tensor([self.batch_size, self.vae_in_channels, 1, spatial, spatial])
+        if input_name == "latent_sample":
             return self.random_float_tensor(
                 [self.batch_size, self.num_channels, self.num_frames, self.height, self.width]
             )
@@ -890,6 +897,8 @@ class LTX2TransformerDummyInputGenerator(DummyVisionInputGenerator):
         "audio_coords",
         "audio_encoder_hidden_states",
         "audio_encoder_attention_mask",
+        "timestep",
+        "audio_timestep",
     )
 
     def __init__(
@@ -948,6 +957,13 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
             return self.random_float_tensor([self.batch_size, self.encoder_seq_length, self.caption_channels])
         if input_name == "audio_encoder_attention_mask":
             return self.random_float_tensor([self.batch_size, self.encoder_seq_length])
+        if input_name == "timestep":
+            # Per-token [B, video_sequence_length]: i2v locks the first frame via the conditioning mask.
+            seq_len = self.num_frames * self.height * self.width
+            return self.random_float_tensor([self.batch_size, seq_len], framework=framework, dtype=float_dtype)
+        if input_name == "audio_timestep":
+            # Audio uses a scalar-per-batch [B] timestep (not per-token, unlike video).
+            return self.random_float_tensor([self.batch_size], framework=framework, dtype=float_dtype)
         return super().generate(input_name, framework, int_dtype, float_dtype)
 
 

optimum/exporters/openvino/model_configs.py

@@ -363,6 +363,10 @@ def init_model_configs():
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"] = {}
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"]["ltx-video"] = "LTXPipeline"
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"]["ltx2"] = "LTX2Pipeline"
+    if is_diffusers_available() and "image-to-video" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS:
+        TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["image-to-video"] = {}
+        TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["image-to-video"]["ltx-video"] = "LTXImageToVideoPipeline"
+        TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["image-to-video"]["ltx2"] = "LTX2ImageToVideoPipeline"
 
 
 init_model_configs()
@@ -2735,7 +2739,8 @@ def inputs(self):
             "num_frames": {},
             "fps": {},
             "audio_num_frames": {},
-            "timestep": {0: "batch_size"},
+            "timestep": {0: "batch_size", 1: "video_sequence_length"},
+            "audio_timestep": {0: "batch_size"},
             "video_coords": {0: "batch_size", 2: "video_sequence_length"},
             "audio_coords": {0: "batch_size", 2: "audio_sequence_length"},
         }

optimum/intel/__init__.py

@@ -134,6 +134,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind
             "OVLTXImageToVideoPipeline",
             "OVLTXPipeline",
             "OVLTX2Pipeline",
+            "OVLTX2ImageToVideoPipeline",
             "OVFluxPipeline",
             "OVFlux2KleinPipeline",
             "OVFluxImg2ImgPipeline",
@@ -165,6 +166,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind
             "OVLTXImageToVideoPipeline",
             "OVLTXPipeline",
             "OVLTX2Pipeline",
+            "OVLTX2ImageToVideoPipeline",
             "OVFluxPipeline",
             "OVFlux2KleinPipeline",
             "OVFluxImg2ImgPipeline",
@@ -250,6 +252,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind
             OVFluxPipeline,
             OVLatentConsistencyModelImg2ImgPipeline,
             OVLatentConsistencyModelPipeline,
+            OVLTX2ImageToVideoPipeline,
             OVLTXImageToVideoPipeline,
             OVPipelineForImage2Image,
             OVPipelineForImage2Video,
@@ -273,6 +276,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind
             OVFluxPipeline,
             OVLatentConsistencyModelImg2ImgPipeline,
             OVLatentConsistencyModelPipeline,
+            OVLTX2ImageToVideoPipeline,
             OVLTXImageToVideoPipeline,
             OVPipelineForImage2Image,
             OVPipelineForImage2Video,

optimum/intel/openvino/__init__.py

@@ -108,6 +108,7 @@
         OVFluxPipeline,
         OVLatentConsistencyModelImg2ImgPipeline,
         OVLatentConsistencyModelPipeline,
+        OVLTX2ImageToVideoPipeline,
         OVLTX2Pipeline,
         OVLTXImageToVideoPipeline,
         OVLTXPipeline,

optimum/intel/openvino/modeling_diffusion.py

@@ -93,9 +93,10 @@
     LTXImageToVideoPipeline = object
 
 if is_diffusers_version(">=", "0.38.0"):
-    from diffusers import LTX2Pipeline
+    from diffusers import LTX2ImageToVideoPipeline, LTX2Pipeline
 else:
     LTX2Pipeline = object
+    LTX2ImageToVideoPipeline = object
 
 if is_diffusers_version(">=", "0.29.0"):
     from diffusers import StableDiffusion3Img2ImgPipeline, StableDiffusion3Pipeline
@@ -1386,12 +1387,30 @@ def forward(
     ):
         self.compile()
 
+        # T2V leaves audio_timestep None; mirror the diffusers fallback before `timestep` is broadcast.
+        if audio_timestep is None:
+            audio_timestep = timestep if timestep is None or timestep.ndim == 1 else timestep[:, 0]
+
+        # T2V passes a scalar timestep [B]; the IR expects [B, S]. Broadcast to match.
+        if timestep is not None and timestep.ndim == 1 and self._timestep_rank == 2:
+            timestep = timestep.unsqueeze(-1).expand(-1, hidden_states.shape[1]).contiguous()
+
+        # `share_inputs=True` reads raw buffers, so stride-0 views (e.g. `t.expand(batch)`) must be materialized.
+        if timestep is not None:
+            timestep = timestep.contiguous()
+        if audio_timestep is not None:
+            audio_timestep = audio_timestep.contiguous()
+
         model_inputs = {
             "hidden_states": hidden_states,
             "timestep": timestep,
             "encoder_hidden_states": encoder_hidden_states,
         }
 
+        # Older t2v exports have no `audio_timestep` input; only pass it when the IR declares it.
+        if audio_timestep is not None and "audio_timestep" in self._ov_input_names:
+            model_inputs["audio_timestep"] = audio_timestep
+
         if audio_hidden_states is not None:
             model_inputs["audio_hidden_states"] = audio_hidden_states
         if audio_encoder_hidden_states is not None:
@@ -1428,6 +1447,18 @@ def forward(
 
         return (model_outputs.get("out_sample"), model_outputs.get("audio_out_sample"))
 
+    @property
+    def _ov_input_names(self):
+        return {inp.get_any_name() for inp in self.model.inputs}
+
+    @property
+    def _timestep_rank(self):
+        # Exact match: "timestep" also matches "audio_timestep" (rank 1), which would break the T2V broadcast.
+        for inp in self.model.inputs:
+            if inp.get_any_name() == "timestep":
+                return len(inp.partial_shape)
+        return 1
+
 
 class OVModelConnectors(OVPipelinePart):
     def forward(
@@ -1947,12 +1978,33 @@ def __call__(self, image=None, **kwargs):
         return super().__call__(image=image, **kwargs)
 
 
-class OVLTX2Pipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, LTX2Pipeline):
-    main_input_name = "prompt"
-    export_feature = "text-to-video"
-    auto_model_class = LTX2Pipeline
+class _OVLTX2Base(OVDiffusionPipeline, OVTextualInversionLoaderMixin):
+    # Shared base rather than i2v subclassing t2v: LTX2ImageToVideoPipeline does not subclass
+    # LTX2Pipeline, so subclassing would resolve `prepare_latents` to the text-to-video version.
     _is_ltx_pipeline = True
 
+    @classproperty
+    def _all_ov_model_paths(cls) -> Dict[str, str]:
+        models_paths = {
+            "transformer": os.path.join(DIFFUSION_MODEL_TRANSFORMER_SUBFOLDER, OV_XML_FILE_NAME),
+            "vae_decoder": os.path.join(DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, OV_XML_FILE_NAME),
+            "text_encoder": os.path.join(DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, OV_XML_FILE_NAME),
+            "connectors": os.path.join(DIFFUSION_MODEL_CONNECTORS_SUBFOLDER, OV_XML_FILE_NAME),
+            "audio_vae_decoder": os.path.join(DIFFUSION_MODEL_AUDIO_VAE_DECODER_SUBFOLDER, OV_XML_FILE_NAME),
+            "vocoder": os.path.join(DIFFUSION_MODEL_VOCODER_SUBFOLDER, OV_XML_FILE_NAME),
+        }
+        return models_paths
+
+    @property
+    def _ov_model_names(self) -> List[str]:
+        """Return list of OV submodel names for quantization."""
+        return list(self._all_ov_model_paths.keys())
+
+    @property
+    def ov_models(self) -> Dict[str, Union[openvino.Model, openvino.CompiledModel]]:
+        """Return dict mapping submodel names to their OV models for quantization."""
+        return {name: component.model for name, component in self.components.items()}
+
     def __init__(
         self,
         scheduler: SchedulerMixin,
@@ -1962,6 +2014,7 @@ def __init__(
         connectors: Optional[openvino.Model] = None,
         audio_vae_decoder: Optional[openvino.Model] = None,
         vocoder: Optional[openvino.Model] = None,
+        vae_encoder: Optional[openvino.Model] = None,
         tokenizer: Optional[CLIPTokenizer] = None,
         device: str = "CPU",
         compile: bool = True,
@@ -2031,7 +2084,12 @@ def __init__(
         )
 
         self.vae_decoder = OVModelVaeDecoder(vae_decoder, self, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER)
-        self.vae_encoder = None
+        # vae_encoder is only exported/loaded for image-to-video; text-to-video leaves it as None.
+        self.vae_encoder = (
+            OVModelVaeEncoder(vae_encoder, self, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER)
+            if isinstance(vae_encoder, openvino.Model)
+            else None
+        )
         self.text_encoder = (
             OVModelTextEncoder(text_encoder, self, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER)
             if text_encoder is not None
@@ -2089,7 +2147,7 @@ def __init__(
             "transformer": self.transformer,
             "vocoder": vocoder,
         }
-        LTX2Pipeline.__init__(self, **diffusers_pipeline_args)
+        self.auto_model_class.__init__(self, **diffusers_pipeline_args)
 
         # This must exist because properties like batch_size check them
         self.unet = None
@@ -2120,6 +2178,8 @@ def components(self) -> Dict[str, Any]:
             comp["transformer"] = self.transformer
         if self.vae_decoder is not None:
             comp["vae_decoder"] = self.vae_decoder
+        if self.vae_encoder is not None:
+            comp["vae_encoder"] = self.vae_encoder
         if self.text_encoder is not None:
             comp["text_encoder"] = self.text_encoder
         if self.connectors is not None:
@@ -2164,6 +2224,12 @@ def reshape(self, batch_size, height, width, num_images_per_prompt=-1, num_frame
         self.vae_decoder.model = self._reshape_vae_decoder(
             self.vae_decoder.model, height, width, num_images_per_prompt, num_frames=num_frames
         )
+        # Reshape vae_encoder (image-to-video only) with pixel-space height/width; the encoder
+        # treats the conditioning image as a single frame (num_frames handled inside).
+        if self.vae_encoder is not None:
+            self.vae_encoder.model = self._reshape_vae_encoder(
+                self.vae_encoder.model, batch_size, height, width, num_frames=num_frames
+            )
         # Reshape text_encoder with batch_size only (tokenizer_max_length stays dynamic for Gemma)
         if self.text_encoder is not None:
             self.text_encoder.model = self._reshape_text_encoder(self.text_encoder.model, batch_size, -1)
@@ -2185,6 +2251,26 @@ def reshape(self, batch_size, height, width, num_images_per_prompt=-1, num_frame
         self.clear_requests()
 
 
+class OVLTX2Pipeline(_OVLTX2Base, LTX2Pipeline):
+    main_input_name = "prompt"
+    export_feature = "text-to-video"
+    auto_model_class = LTX2Pipeline
+
+
+class OVLTX2ImageToVideoPipeline(_OVLTX2Base, LTX2ImageToVideoPipeline):
+    main_input_name = "image"
+    export_feature = "image-to-video"
+    auto_model_class = LTX2ImageToVideoPipeline
+    _vae_encoder_single_frame = True
+
+    @classproperty
+    def _all_ov_model_paths(cls) -> Dict[str, str]:
+        # Same submodels as text-to-video, plus the VAE encoder used to encode the input image.
+        models_paths = _OVLTX2Base._all_ov_model_paths
+        models_paths["vae_encoder"] = os.path.join(DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, OV_XML_FILE_NAME)
+        return models_paths
+
+
 SUPPORTED_OV_PIPELINES = [
     OVStableDiffusionPipeline,
     OVStableDiffusionImg2ImgPipeline,
@@ -2243,7 +2329,9 @@ def _get_ov_class(pipeline_class_name: str, throw_error_if_not_exist: bool = Tru
 
 if is_diffusers_version(">=", "0.38.0"):
     OV_TEXT2VIDEO_PIPELINES_MAPPING["ltx2"] = OVLTX2Pipeline
+    OV_IMAGE2VIDEO_PIPELINES_MAPPING["ltx2"] = OVLTX2ImageToVideoPipeline
     SUPPORTED_OV_PIPELINES.append(OVLTX2Pipeline)
+    SUPPORTED_OV_PIPELINES.append(OVLTX2ImageToVideoPipeline)
 
 if is_diffusers_version(">=", "0.29.0"):
     SUPPORTED_OV_PIPELINES.extend(

optimum/intel/utils/dummy_openvino_and_diffusers_objects.py

@@ -136,6 +136,17 @@ def from_pretrained(cls, *args, **kwargs):
         requires_backends(cls, ["openvino", "diffusers"])
 
 
+class OVLTX2ImageToVideoPipeline(metaclass=DummyObject):
+    _backends = ["openvino", "diffusers"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["openvino", "diffusers"])
+
+    @classmethod
+    def from_pretrained(cls, *args, **kwargs):
+        requires_backends(cls, ["openvino", "diffusers"])
+
+
 class OVDiffusionPipeline(metaclass=DummyObject):
     _backends = ["openvino", "diffusers"]
 

tests/openvino/test_diffusion.py

@@ -1215,6 +1215,8 @@ class OVPipelineForImage2VideoTest(unittest.TestCase):
     SUPPORTED_ARCHITECTURES = []
     if is_diffusers_version(">=", "0.32"):
         SUPPORTED_ARCHITECTURES.extend(["ltx-video"])
+    if is_diffusers_version(">=", "0.38.0"):
+        SUPPORTED_ARCHITECTURES.extend(["ltx2"])
 
     OVMODEL_CLASS = OVPipelineForImage2Video
     AUTOMODEL_CLASS = DiffusionPipeline
@@ -1242,12 +1244,20 @@ def test_load_vanilla_model_which_is_not_supported(self):
 
         self.assertIn(f"does not appear to have a file named {self.OVMODEL_CLASS.config_name}", str(context.exception))
 
+    @staticmethod
+    def _auto_cls(model_arch: str):
+        if model_arch == "ltx2":
+            from diffusers import LTX2ImageToVideoPipeline
+
+            return LTX2ImageToVideoPipeline
+        from diffusers import LTXImageToVideoPipeline
+
+        return LTXImageToVideoPipeline
+
     @parameterized.expand(SUPPORTED_ARCHITECTURES, skip_on_empty=True)
     @require_diffusers
     def test_ov_pipeline_class_dispatch(self, model_arch: str):
-        from diffusers import LTXImageToVideoPipeline
-
-        auto_cls = LTXImageToVideoPipeline
+        auto_cls = self._auto_cls(model_arch)
         auto_pipeline = auto_cls.from_pretrained(MODEL_NAMES[model_arch])
         ov_pipeline = self.OVMODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], device=OPENVINO_DEVICE)
 
@@ -1270,12 +1280,10 @@ def test_num_videos_per_prompt(self, model_arch: str):
     @parameterized.expand(SUPPORTED_ARCHITECTURES, skip_on_empty=True)
     @require_diffusers
     def test_compare_to_diffusers_pipeline(self, model_arch: str):
-        from diffusers import LTXImageToVideoPipeline
-
         height, width, batch_size = 64, 96, 1
         inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size)
         ov_pipeline = self.OVMODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], device=OPENVINO_DEVICE)
-        diffusers_pipeline = LTXImageToVideoPipeline.from_pretrained(MODEL_NAMES[model_arch])
+        diffusers_pipeline = self._auto_cls(model_arch).from_pretrained(MODEL_NAMES[model_arch])
 
         for output_type in ["np", "pt"]:
             inputs["output_type"] = output_type

tests/openvino/test_exporters_cli.py

@@ -113,6 +113,7 @@ class OVCLIExportTestCase(unittest.TestCase):
         ("text-to-image", "sana"),
         ("text-to-video", "ltx-video"),
         ("text-to-video", "ltx2"),
+        ("image-to-video", "ltx2"),
         ("feature-extraction", "sam"),
         ("text-to-audio", "speecht5"),
         ("zero-shot-image-classification", "clip"),