diff --git a/demo/vibevoice_asr_inference_from_file.py b/demo/vibevoice_asr_inference_from_file.py index bf4f75df..497ca743 100644 --- a/demo/vibevoice_asr_inference_from_file.py +++ b/demo/vibevoice_asr_inference_from_file.py @@ -54,12 +54,12 @@ def __init__( print(f"Using attention implementation: {attn_implementation}") self.model = VibeVoiceASRForConditionalGeneration.from_pretrained( model_path, - dtype=dtype, + torch_dtype=dtype, device_map=device if device == "auto" else None, attn_implementation=attn_implementation, trust_remote_code=True ) - + if device != "auto": self.model = self.model.to(device) @@ -443,7 +443,7 @@ def main(): parser.add_argument( "--device", type=str, - default="cuda" if torch.cuda.is_available() else ("xpu" if torch.backends.xpu.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu") ), + default="cuda" if torch.cuda.is_available() else ("xpu" if hasattr(torch.backends, 'xpu') and torch.backends.xpu.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu") ), choices=["cuda", "cpu", "mps","xpu", "auto"], help="Device to run inference on" ) @@ -533,7 +533,7 @@ def main(): # Initialize model # Handle MPS device and dtype if args.device == "mps": - model_dtype = torch.float32 # MPS works better with float32 + model_dtype = torch.float16 # float16 to fit within MPS memory (~14GB vs ~28GB for float32) elif args.device == "xpu": model_dtype = torch.float32 elif args.device == "cpu": diff --git a/vibevoice/modular/configuration_vibevoice.py b/vibevoice/modular/configuration_vibevoice.py index 18451136..141f9ce3 100644 --- a/vibevoice/modular/configuration_vibevoice.py +++ b/vibevoice/modular/configuration_vibevoice.py @@ -14,17 +14,15 @@ def _convert_dtype_to_string(config_dict: dict) -> dict: """ Convert torch.dtype objects to their string representation for JSON serialization. - - This fixes the "Object of type dtype is not JSON serializable" error that occurs - when transformers tries to log/serialize the config with torch_dtype as a torch.dtype object. - + Recursively handles nested dicts. + See: https://github.com/microsoft/VibeVoice/issues/199 """ - if "torch_dtype" in config_dict and config_dict["torch_dtype"] is not None: - dtype = config_dict["torch_dtype"] - if isinstance(dtype, torch.dtype): - # Convert torch.dtype to string (e.g., torch.bfloat16 -> "bfloat16") - config_dict["torch_dtype"] = str(dtype).replace("torch.", "") + for key, value in config_dict.items(): + if isinstance(value, torch.dtype): + config_dict[key] = str(value).replace("torch.", "") + elif isinstance(value, dict): + config_dict[key] = _convert_dtype_to_string(value) return config_dict