diff --git a/docs/source/openvino/inference.mdx b/docs/source/openvino/inference.mdx index 6153303eb0..20ef7aedc3 100644 --- a/docs/source/openvino/inference.mdx +++ b/docs/source/openvino/inference.mdx @@ -80,8 +80,10 @@ As shown in the table below, each task is associated with a class enabling to au | `OVStableDiffusionXLImg2ImgPipeline` | `image-to-image` | | `OVLatentConsistencyModelPipeline` | `text-to-image` | | `OVLTXPipeline` | `text-to-video` | +| `OVLTX2Pipeline` | `text-to-video` | | `OVPipelineForText2Video` | `text-to-video` | | `OVLTXImageToVideoPipeline` | `image-to-video` | +| `OVLTX2ImageToVideoPipeline` | `image-to-video` | | `OVPipelineForImage2Video` | `image-to-video` | See the [reference documentation](reference) for more information about parameters, and examples for different tasks. diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index b2c375048a..7597b5ef55 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -1295,6 +1295,22 @@ def get_ltx2_video_models_for_export(pipeline, exporter, int_dtype, float_dtype) vae_decoder_export_config.runtime_options = {"ACTIVATIONS_SCALE_FACTOR": "8.0"} models_for_export["vae_decoder"] = (vae_decoder, vae_decoder_export_config) + # VAE Encoder (needed for image-to-video conditioning; harmless for text-to-video, which won't load it) + vae_encoder = copy.deepcopy(pipeline.vae) + vae_encoder.forward = lambda sample: {"latent_parameters": vae_encoder.encode(x=sample)["latent_dist"].parameters} + vae_encoder_config_constructor = TasksManager.get_exporter_config_constructor( + model=vae_encoder, + exporter=exporter, + library_name="diffusers", + task="semantic-segmentation", + model_type="ltx2-vae-encoder", + ) + vae_encoder_export_config = vae_encoder_config_constructor( + vae_encoder.config, int_dtype=int_dtype, float_dtype=float_dtype + ) + vae_encoder_export_config.runtime_options = {"ACTIVATIONS_SCALE_FACTOR": "8.0"} + models_for_export["vae_encoder"] = (vae_encoder, vae_encoder_export_config) + # Audio VAE decoder and vocoder if hasattr(pipeline, "audio_vae") and pipeline.audio_vae is not None: # Audio VAE decoder diff --git a/optimum/exporters/openvino/input_generators.py b/optimum/exporters/openvino/input_generators.py index 60d485804f..ee156c1815 100644 --- a/optimum/exporters/openvino/input_generators.py +++ b/optimum/exporters/openvino/input_generators.py @@ -867,9 +867,16 @@ def __init__( ): super().__init__(task, normalized_config, batch_size, num_channels, width, height, **kwargs) self.num_frames = num_frames + # Pixel-space properties used to shape the VAE encoder input (`sample`). + self.vae_in_channels = getattr(normalized_config.config, "in_channels", 3) + self.spatial_compression_ratio = getattr(normalized_config.config, "spatial_compression_ratio", 32) def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): - if input_name in ["sample", "latent_sample"]: + if input_name == "sample": + # Pixel-space input with a single conditioning frame; num_frames=1 satisfies the temporal patchify. + spatial = self.spatial_compression_ratio + return self.random_float_tensor([self.batch_size, self.vae_in_channels, 1, spatial, spatial]) + if input_name == "latent_sample": return self.random_float_tensor( [self.batch_size, self.num_channels, self.num_frames, self.height, self.width] ) @@ -892,6 +899,8 @@ class LTX2TransformerDummyInputGenerator(DummyVisionInputGenerator): "audio_coords", "audio_encoder_hidden_states", "audio_encoder_attention_mask", + "timestep", + "audio_timestep", ) def __init__( @@ -950,6 +959,13 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int return self.random_float_tensor([self.batch_size, self.encoder_seq_length, self.caption_channels]) if input_name == "audio_encoder_attention_mask": return self.random_float_tensor([self.batch_size, self.encoder_seq_length]) + if input_name == "timestep": + # Per-token [B, video_sequence_length]: i2v locks the first frame via the conditioning mask. + seq_len = self.num_frames * self.height * self.width + return self.random_float_tensor([self.batch_size, seq_len], framework=framework, dtype=float_dtype) + if input_name == "audio_timestep": + # Audio uses a scalar-per-batch [B] timestep (not per-token, unlike video). + return self.random_float_tensor([self.batch_size], framework=framework, dtype=float_dtype) return super().generate(input_name, framework, int_dtype, float_dtype) diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index a09324d3d4..bb48b96e2c 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -365,10 +365,13 @@ def init_model_configs(): TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-image"] = {} TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-image"]["sana"] = "SanaPipeline" TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-image"]["sana-sprint"] = "SanaSprintPipeline" - if is_diffusers_available() and "text-to-video" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS: - TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"] = {} + if is_diffusers_available(): + TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS.setdefault("text-to-video", {}) TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"]["ltx-video"] = "LTXPipeline" TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["text-to-video"]["ltx2"] = "LTX2Pipeline" + TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS.setdefault("image-to-video", {}) + TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["image-to-video"]["ltx-video"] = "LTXImageToVideoPipeline" + TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["image-to-video"]["ltx2"] = "LTX2ImageToVideoPipeline" init_model_configs() @@ -2935,7 +2938,8 @@ def inputs(self): "num_frames": {}, "fps": {}, "audio_num_frames": {}, - "timestep": {0: "batch_size"}, + "timestep": {0: "batch_size", 1: "video_sequence_length"}, + "audio_timestep": {0: "batch_size"}, "video_coords": {0: "batch_size", 2: "video_sequence_length"}, "audio_coords": {0: "batch_size", 2: "audio_sequence_length"}, } diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index 36c692a782..4e7d085cea 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -134,6 +134,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind "OVLTXImageToVideoPipeline", "OVLTXPipeline", "OVLTX2Pipeline", + "OVLTX2ImageToVideoPipeline", "OVFluxPipeline", "OVFlux2KleinPipeline", "OVFluxImg2ImgPipeline", @@ -166,6 +167,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind "OVLTXImageToVideoPipeline", "OVLTXPipeline", "OVLTX2Pipeline", + "OVLTX2ImageToVideoPipeline", "OVFluxPipeline", "OVFlux2KleinPipeline", "OVFluxImg2ImgPipeline", @@ -252,6 +254,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind OVFluxPipeline, OVLatentConsistencyModelImg2ImgPipeline, OVLatentConsistencyModelPipeline, + OVLTX2ImageToVideoPipeline, OVLTXImageToVideoPipeline, OVPipelineForImage2Image, OVPipelineForImage2Video, @@ -276,6 +279,7 @@ def _patched_code_predictor_init(self, *args, use_sliding_window=False, max_wind OVFluxPipeline, OVLatentConsistencyModelImg2ImgPipeline, OVLatentConsistencyModelPipeline, + OVLTX2ImageToVideoPipeline, OVLTXImageToVideoPipeline, OVPipelineForImage2Image, OVPipelineForImage2Video, diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 314381e879..f95a002bda 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -108,6 +108,7 @@ OVFluxPipeline, OVLatentConsistencyModelImg2ImgPipeline, OVLatentConsistencyModelPipeline, + OVLTX2ImageToVideoPipeline, OVLTX2Pipeline, OVLTXImageToVideoPipeline, OVLTXPipeline, diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index b40733e5de..6500db6623 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -93,9 +93,10 @@ LTXImageToVideoPipeline = object if is_diffusers_version(">=", "0.38.0"): - from diffusers import LTX2Pipeline + from diffusers import LTX2ImageToVideoPipeline, LTX2Pipeline else: LTX2Pipeline = object + LTX2ImageToVideoPipeline = object if is_diffusers_version(">=", "0.29.0"): from diffusers import StableDiffusion3Img2ImgPipeline, StableDiffusion3Pipeline @@ -1462,12 +1463,30 @@ def forward( ): self.compile() + # T2V leaves audio_timestep None; mirror the diffusers fallback before `timestep` is broadcast. + if audio_timestep is None: + audio_timestep = timestep if timestep is None or timestep.ndim == 1 else timestep[:, 0] + + # T2V passes a scalar timestep [B]; the IR expects [B, S]. Broadcast to match. + if timestep is not None and timestep.ndim == 1 and self._timestep_rank == 2: + timestep = timestep.unsqueeze(-1).expand(-1, hidden_states.shape[1]).contiguous() + + # `share_inputs=True` reads raw buffers, so stride-0 views (e.g. `t.expand(batch)`) must be materialized. + if timestep is not None: + timestep = timestep.contiguous() + if audio_timestep is not None: + audio_timestep = audio_timestep.contiguous() + model_inputs = { "hidden_states": hidden_states, "timestep": timestep, "encoder_hidden_states": encoder_hidden_states, } + # Older t2v exports have no `audio_timestep` input; only pass it when the IR declares it. + if audio_timestep is not None and "audio_timestep" in self._ov_input_names: + model_inputs["audio_timestep"] = audio_timestep + if audio_hidden_states is not None: model_inputs["audio_hidden_states"] = audio_hidden_states if audio_encoder_hidden_states is not None: @@ -1504,6 +1523,18 @@ def forward( return (model_outputs.get("out_sample"), model_outputs.get("audio_out_sample")) + @property + def _ov_input_names(self): + return {inp.get_any_name() for inp in self.model.inputs} + + @property + def _timestep_rank(self): + # Exact match: "timestep" also matches "audio_timestep" (rank 1), which would break the T2V broadcast. + for inp in self.model.inputs: + if inp.get_any_name() == "timestep": + return len(inp.partial_shape) + return 1 + class OVModelConnectors(OVPipelinePart): def forward( @@ -2028,12 +2059,33 @@ def __call__(self, image=None, **kwargs): return super().__call__(image=image, **kwargs) -class OVLTX2Pipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, LTX2Pipeline): - main_input_name = "prompt" - export_feature = "text-to-video" - auto_model_class = LTX2Pipeline +class _OVLTX2Base(OVDiffusionPipeline, OVTextualInversionLoaderMixin): + # Shared base rather than i2v subclassing t2v: LTX2ImageToVideoPipeline does not subclass + # LTX2Pipeline, so subclassing would resolve `prepare_latents` to the text-to-video version. _is_ltx_pipeline = True + @classproperty + def _all_ov_model_paths(cls) -> Dict[str, str]: + models_paths = { + "transformer": os.path.join(DIFFUSION_MODEL_TRANSFORMER_SUBFOLDER, OV_XML_FILE_NAME), + "vae_decoder": os.path.join(DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, OV_XML_FILE_NAME), + "text_encoder": os.path.join(DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, OV_XML_FILE_NAME), + "connectors": os.path.join(DIFFUSION_MODEL_CONNECTORS_SUBFOLDER, OV_XML_FILE_NAME), + "audio_vae_decoder": os.path.join(DIFFUSION_MODEL_AUDIO_VAE_DECODER_SUBFOLDER, OV_XML_FILE_NAME), + "vocoder": os.path.join(DIFFUSION_MODEL_VOCODER_SUBFOLDER, OV_XML_FILE_NAME), + } + return models_paths + + @property + def _ov_model_names(self) -> List[str]: + """Return list of OV submodel names for quantization.""" + return list(self._all_ov_model_paths.keys()) + + @property + def ov_models(self) -> Dict[str, Union[openvino.Model, openvino.CompiledModel]]: + """Return dict mapping submodel names to their OV models for quantization.""" + return {name: component.model for name, component in self.components.items()} + def __init__( self, scheduler: SchedulerMixin, @@ -2043,6 +2095,7 @@ def __init__( connectors: Optional[openvino.Model] = None, audio_vae_decoder: Optional[openvino.Model] = None, vocoder: Optional[openvino.Model] = None, + vae_encoder: Optional[openvino.Model] = None, tokenizer: Optional[CLIPTokenizer] = None, device: str = "CPU", compile: bool = True, @@ -2112,7 +2165,12 @@ def __init__( ) self.vae_decoder = OVModelVaeDecoder(vae_decoder, self, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER) - self.vae_encoder = None + # vae_encoder is exported for both tasks but only loaded for image-to-video (None otherwise). + self.vae_encoder = ( + OVModelVaeEncoder(vae_encoder, self, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER) + if isinstance(vae_encoder, openvino.Model) + else None + ) self.text_encoder = ( OVModelTextEncoder(text_encoder, self, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER) if text_encoder is not None @@ -2170,7 +2228,7 @@ def __init__( "transformer": self.transformer, "vocoder": vocoder, } - LTX2Pipeline.__init__(self, **diffusers_pipeline_args) + self.auto_model_class.__init__(self, **diffusers_pipeline_args) # This must exist because properties like batch_size check them self.unet = None @@ -2201,6 +2259,8 @@ def components(self) -> Dict[str, Any]: comp["transformer"] = self.transformer if self.vae_decoder is not None: comp["vae_decoder"] = self.vae_decoder + if self.vae_encoder is not None: + comp["vae_encoder"] = self.vae_encoder if self.text_encoder is not None: comp["text_encoder"] = self.text_encoder if self.connectors is not None: @@ -2245,6 +2305,12 @@ def reshape(self, batch_size, height, width, num_images_per_prompt=-1, num_frame self.vae_decoder.model = self._reshape_vae_decoder( self.vae_decoder.model, height, width, num_images_per_prompt, num_frames=num_frames ) + # Reshape vae_encoder (image-to-video only) with pixel-space height/width; the encoder + # treats the conditioning image as a single frame (num_frames handled inside). + if self.vae_encoder is not None: + self.vae_encoder.model = self._reshape_vae_encoder( + self.vae_encoder.model, batch_size, height, width, num_frames=num_frames + ) # Reshape text_encoder with batch_size only (tokenizer_max_length stays dynamic for Gemma) if self.text_encoder is not None: self.text_encoder.model = self._reshape_text_encoder(self.text_encoder.model, batch_size, -1) @@ -2266,6 +2332,26 @@ def reshape(self, batch_size, height, width, num_images_per_prompt=-1, num_frame self.clear_requests() +class OVLTX2Pipeline(_OVLTX2Base, LTX2Pipeline): + main_input_name = "prompt" + export_feature = "text-to-video" + auto_model_class = LTX2Pipeline + + +class OVLTX2ImageToVideoPipeline(_OVLTX2Base, LTX2ImageToVideoPipeline): + main_input_name = "image" + export_feature = "image-to-video" + auto_model_class = LTX2ImageToVideoPipeline + _vae_encoder_single_frame = True + + @classproperty + def _all_ov_model_paths(cls) -> Dict[str, str]: + # Same submodels as text-to-video, plus the VAE encoder used to encode the input image. + models_paths = _OVLTX2Base._all_ov_model_paths + models_paths["vae_encoder"] = os.path.join(DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, OV_XML_FILE_NAME) + return models_paths + + class OVQwenImagePipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, QwenImagePipeline): main_input_name = "prompt" export_feature = "text-to-image" @@ -2370,7 +2456,9 @@ def _get_ov_class(pipeline_class_name: str, throw_error_if_not_exist: bool = Tru if is_diffusers_version(">=", "0.38.0"): OV_TEXT2VIDEO_PIPELINES_MAPPING["ltx2"] = OVLTX2Pipeline + OV_IMAGE2VIDEO_PIPELINES_MAPPING["ltx2"] = OVLTX2ImageToVideoPipeline SUPPORTED_OV_PIPELINES.append(OVLTX2Pipeline) + SUPPORTED_OV_PIPELINES.append(OVLTX2ImageToVideoPipeline) if is_diffusers_version(">=", "0.29.0"): SUPPORTED_OV_PIPELINES.extend( diff --git a/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py b/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py index 33874bf35c..e5201ef036 100644 --- a/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py +++ b/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py @@ -136,6 +136,17 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["openvino", "diffusers"]) +class OVLTX2ImageToVideoPipeline(metaclass=DummyObject): + _backends = ["openvino", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino", "diffusers"]) + + class OVDiffusionPipeline(metaclass=DummyObject): _backends = ["openvino", "diffusers"] diff --git a/tests/openvino/test_diffusion.py b/tests/openvino/test_diffusion.py index 86c4ecdb00..d8e59e8813 100644 --- a/tests/openvino/test_diffusion.py +++ b/tests/openvino/test_diffusion.py @@ -1219,6 +1219,8 @@ class OVPipelineForImage2VideoTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = [] if is_diffusers_version(">=", "0.32"): SUPPORTED_ARCHITECTURES.extend(["ltx-video"]) + if is_diffusers_version(">=", "0.38.0"): + SUPPORTED_ARCHITECTURES.extend(["ltx2"]) OVMODEL_CLASS = OVPipelineForImage2Video AUTOMODEL_CLASS = DiffusionPipeline @@ -1246,12 +1248,20 @@ def test_load_vanilla_model_which_is_not_supported(self): self.assertIn(f"does not appear to have a file named {self.OVMODEL_CLASS.config_name}", str(context.exception)) + @staticmethod + def _auto_cls(model_arch: str): + if model_arch == "ltx2": + from diffusers import LTX2ImageToVideoPipeline + + return LTX2ImageToVideoPipeline + from diffusers import LTXImageToVideoPipeline + + return LTXImageToVideoPipeline + @parameterized.expand(SUPPORTED_ARCHITECTURES, skip_on_empty=True) @require_diffusers def test_ov_pipeline_class_dispatch(self, model_arch: str): - from diffusers import LTXImageToVideoPipeline - - auto_cls = LTXImageToVideoPipeline + auto_cls = self._auto_cls(model_arch) auto_pipeline = auto_cls.from_pretrained(MODEL_NAMES[model_arch]) ov_pipeline = self.OVMODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], device=OPENVINO_DEVICE) @@ -1274,12 +1284,10 @@ def test_num_videos_per_prompt(self, model_arch: str): @parameterized.expand(SUPPORTED_ARCHITECTURES, skip_on_empty=True) @require_diffusers def test_compare_to_diffusers_pipeline(self, model_arch: str): - from diffusers import LTXImageToVideoPipeline - height, width, batch_size = 64, 96, 1 inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) ov_pipeline = self.OVMODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], device=OPENVINO_DEVICE) - diffusers_pipeline = LTXImageToVideoPipeline.from_pretrained(MODEL_NAMES[model_arch]) + diffusers_pipeline = self._auto_cls(model_arch).from_pretrained(MODEL_NAMES[model_arch]) for output_type in ["np", "pt"]: inputs["output_type"] = output_type @@ -1309,7 +1317,9 @@ def test_image_reproducibility(self, model_arch: str): pipeline = self.OVMODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], device=OPENVINO_DEVICE) height, width, batch_size = 64, 96, 1 - inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + # I2V keeps the first latent frame as image conditioning, so use a generated frame too. + num_frames = getattr(pipeline, "vae_temporal_compression_ratio", 1) + 1 + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size, num_frames=num_frames) for generator_framework in ["np", "pt"]: ov_outputs_1 = pipeline(**inputs, generator=get_generator(generator_framework, SEED)) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index fd23afe149..273327fd89 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -113,6 +113,7 @@ class OVCLIExportTestCase(unittest.TestCase): ("text-to-image", "sana"), ("text-to-video", "ltx-video"), ("text-to-video", "ltx2"), + ("image-to-video", "ltx2"), ("feature-extraction", "sam"), ("text-to-audio", "speecht5"), ("zero-shot-image-classification", "clip"),