diff --git a/site/docs/supported-models/_components/vlm-models-table/models.ts b/site/docs/supported-models/_components/vlm-models-table/models.ts index acffe74603..0008907596 100644 --- a/site/docs/supported-models/_components/vlm-models-table/models.ts +++ b/site/docs/supported-models/_components/vlm-models-table/models.ts @@ -263,4 +263,15 @@ export const VLM_MODELS: VLMModelType[] = [ }, ], }, + { + architecture: 'YoutuVLForConditionalGeneration', + models: [ + { + name: 'Youtu-VL', + links: [ + 'https://huggingface.co/tencent/Youtu-VL-4B-Instruct', + ], + }, + ], + }, ]; diff --git a/src/cpp/src/visual_language/inputs_embedder.cpp b/src/cpp/src/visual_language/inputs_embedder.cpp index 40660f5bcc..c4b336a25b 100644 --- a/src/cpp/src/visual_language/inputs_embedder.cpp +++ b/src/cpp/src/visual_language/inputs_embedder.cpp @@ -25,6 +25,7 @@ #include "visual_language/gemma3n/classes.hpp" #include "visual_language/gemma4/classes.hpp" #include "visual_language/videochat_flash/classes.hpp" +#include "visual_language/youtu_vl/classes.hpp" #include "continuous_batching/timer.hpp" #include "utils.hpp" @@ -384,6 +385,8 @@ InputsEmbedder::InputsEmbedder(const std::filesystem::path& model_dir, m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); } else if (vlm_config.model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); + } else if (vlm_config.model_type == VLMModelType::YOUTU_VL) { + m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM InputsEmbedder class. Please, create feature request on new model support"); } @@ -432,6 +435,8 @@ InputsEmbedder::InputsEmbedder(const ModelsMap& models_map, m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); } else if (vlm_config.model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); + } else if (vlm_config.model_type == VLMModelType::YOUTU_VL) { + m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM InputsEmbedder class. Please, create feature request on new model support"); } diff --git a/src/cpp/src/visual_language/inputs_embedder.hpp b/src/cpp/src/visual_language/inputs_embedder.hpp index 2d3218dacf..e7a49fd964 100644 --- a/src/cpp/src/visual_language/inputs_embedder.hpp +++ b/src/cpp/src/visual_language/inputs_embedder.hpp @@ -397,6 +397,7 @@ class InputsEmbedder { friend class InputsEmbedderGemma3n; friend class InputsEmbedderGemma4; friend class InputsEmbedderVideoChatFlashQwen; + friend class InputsEmbedderYoutuVL; }; template diff --git a/src/cpp/src/visual_language/vision_encoder.cpp b/src/cpp/src/visual_language/vision_encoder.cpp index 40daf8d967..0c53d4c4a8 100644 --- a/src/cpp/src/visual_language/vision_encoder.cpp +++ b/src/cpp/src/visual_language/vision_encoder.cpp @@ -25,6 +25,7 @@ #include "visual_language/gemma3n/classes.hpp" #include "visual_language/gemma4/classes.hpp" #include "visual_language/videochat_flash/classes.hpp" +#include "visual_language/youtu_vl/classes.hpp" namespace ov::genai { @@ -146,6 +147,8 @@ VisionEncoder::Ptr VisionEncoder::create(const std::filesystem::path& model_dir, return std::make_shared(model_dir, device, properties); } else if (model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { return std::make_shared(model_dir, device, properties); + } else if (model_type == VLMModelType::YOUTU_VL) { + return std::make_shared(model_dir, device, properties); } else { OPENVINO_THROW("Unsupported model type in VLM VisionEncoder class. Please, create feature request on new model support"); } @@ -193,6 +196,8 @@ VisionEncoder::Ptr VisionEncoder::create( return std::make_shared(models_map, config_dir_path, device, device_config); } else if (model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { return std::make_shared(models_map, config_dir_path, device, device_config); + } else if (model_type == VLMModelType::YOUTU_VL) { + return std::make_shared(models_map, config_dir_path, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM VisionEncoder class. Please, create feature request on new model support"); } diff --git a/src/cpp/src/visual_language/vlm_config.cpp b/src/cpp/src/visual_language/vlm_config.cpp index 0846e52bfa..e3c7b918f2 100644 --- a/src/cpp/src/visual_language/vlm_config.cpp +++ b/src/cpp/src/visual_language/vlm_config.cpp @@ -34,6 +34,7 @@ VLMModelType to_vlm_model_type(const std::string& value) { {"videochat_flash_qwen", VLMModelType::VIDEOCHAT_FLASH_QWEN}, {"qwen3_omni", VLMModelType::QWEN3_OMNI}, {"qwen3_omni_moe", VLMModelType::QWEN3_OMNI}, + {"youtu_vl", VLMModelType::YOUTU_VL}, }; auto it = model_types_map.find(value); diff --git a/src/cpp/src/visual_language/vlm_config.hpp b/src/cpp/src/visual_language/vlm_config.hpp index 6552e77199..0e260b42d9 100644 --- a/src/cpp/src/visual_language/vlm_config.hpp +++ b/src/cpp/src/visual_language/vlm_config.hpp @@ -32,6 +32,7 @@ enum class VLMModelType { GEMMA4_UNIFIED, VIDEOCHAT_FLASH_QWEN, QWEN3_OMNI, + YOUTU_VL, }; /// @brief A Configuration class passed to VLMPipeline and used to diff --git a/src/cpp/src/visual_language/youtu_vl/classes.cpp b/src/cpp/src/visual_language/youtu_vl/classes.cpp new file mode 100644 index 0000000000..cedd15b739 --- /dev/null +++ b/src/cpp/src/visual_language/youtu_vl/classes.cpp @@ -0,0 +1,534 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "visual_language/youtu_vl/classes.hpp" + +#include + +#include "visual_language/clip.hpp" +#include "utils.hpp" + +namespace ov::genai { + +namespace { + +// SigLIP2 fast image processor resize rounding: each side is rounded up to a +// multiple of patch_size * 2 and the whole image is scaled down (in 0.02 steps) +// until (H/patch_size) * (W/patch_size) <= max_num_patches. +// Mirrors image_processing_siglip2_fast.get_image_size_for_patches. +size_t get_scaled_image_size(double scale, size_t size, size_t patch_size) { + size_t rounding = patch_size * 2; + double scaled = static_cast(size) * scale; + size_t rounded = static_cast(std::ceil(scaled / static_cast(rounding))) * rounding; + return std::max(rounding, rounded); +} + +std::pair get_target_image_size(size_t height, size_t width, size_t patch_size, size_t max_num_patches) { + double scale = 1.0; + size_t target_height = 0; + size_t target_width = 0; + while (true) { + target_height = get_scaled_image_size(scale, height, patch_size); + target_width = get_scaled_image_size(scale, width, patch_size); + double num_patches = (static_cast(target_height) / patch_size) * + (static_cast(target_width) / patch_size); + if (num_patches > static_cast(max_num_patches)) { + scale -= 0.02; + } else { + break; + } + } + return {target_height, target_width}; +} + +// One-dimensional anti-aliased bilinear (triangle-filter) resampling weights, +// matching torchvision.transforms.functional.resize(..., interpolation=BILINEAR, +// antialias=True), which the SigLIP2 fast image processor uses. On downscaling +// the triangle kernel is stretched by the scale factor so that source pixels are +// averaged (anti-aliasing); on upscaling it degrades to plain bilinear. Each +// output pixel gets a contiguous window [start, start + weights.size()). +struct ResampleDim { + std::vector starts; // per output index: first source index + std::vector> weights; // per output index: normalized weights +}; + +ResampleDim compute_resample_weights(size_t in_size, size_t out_size) { + const double scale = static_cast(in_size) / static_cast(out_size); + const double filterscale = scale > 1.0 ? scale : 1.0; + const double support = filterscale; // bilinear support radius (== 1) times filterscale + + ResampleDim dim; + dim.starts.resize(out_size); + dim.weights.resize(out_size); + + for (size_t ox = 0; ox < out_size; ++ox) { + const double center = (static_cast(ox) + 0.5) * scale; + long xmin = static_cast(std::floor(center - support)); + long xmax = static_cast(std::ceil(center + support)); + if (xmin < 0) { + xmin = 0; + } + if (xmax > static_cast(in_size)) { + xmax = static_cast(in_size); + } + + std::vector ws; + ws.reserve(static_cast(xmax - xmin)); + double sum = 0.0; + for (long x = xmin; x < xmax; ++x) { + const double arg = std::abs((static_cast(x) + 0.5 - center) / filterscale); + const double w = arg < 1.0 ? (1.0 - arg) : 0.0; + ws.push_back(w); + sum += w; + } + if (sum != 0.0) { + for (double& w : ws) { + w /= sum; + } + } + dim.starts[ox] = static_cast(xmin); + dim.weights[ox] = std::move(ws); + } + return dim; +} + +// Anti-aliased bilinear resize of an HWC uint8 image, producing a normalized CHW +// float buffer: value = ((pixel / 255) - mean) / std. Uses a separable two-pass +// resample (horizontal then vertical) to mirror torchvision's implementation. +std::vector antialias_resize_normalize_chw(const clip_image_u8& src, + size_t target_h, + size_t target_w, + const float* image_mean, + const float* image_std) { + const size_t src_h = static_cast(src.ny); + const size_t src_w = static_cast(src.nx); + const size_t channels = 3; + + const ResampleDim wx = compute_resample_weights(src_w, target_w); + const ResampleDim wy = compute_resample_weights(src_h, target_h); + + // Horizontal pass: [src_h, target_w, C] in double precision. + std::vector tmp(src_h * target_w * channels, 0.0); + const uint8_t* src_buf = src.buf.data(); + for (size_t y = 0; y < src_h; ++y) { + for (size_t ox = 0; ox < target_w; ++ox) { + const size_t xstart = wx.starts[ox]; + const std::vector& ws = wx.weights[ox]; + double acc[3] = {0.0, 0.0, 0.0}; + for (size_t k = 0; k < ws.size(); ++k) { + const size_t sx = xstart + k; + const uint8_t* px = src_buf + (y * src_w + sx) * channels; + acc[0] += ws[k] * px[0]; + acc[1] += ws[k] * px[1]; + acc[2] += ws[k] * px[2]; + } + double* dst = tmp.data() + (y * target_w + ox) * channels; + dst[0] = acc[0]; + dst[1] = acc[1]; + dst[2] = acc[2]; + } + } + + // Vertical pass + normalize, writing directly to CHW float layout. + std::vector chw(channels * target_h * target_w); + for (size_t oy = 0; oy < target_h; ++oy) { + const size_t ystart = wy.starts[oy]; + const std::vector& ws = wy.weights[oy]; + for (size_t ox = 0; ox < target_w; ++ox) { + double acc[3] = {0.0, 0.0, 0.0}; + for (size_t k = 0; k < ws.size(); ++k) { + const size_t sy = ystart + k; + const double* px = tmp.data() + (sy * target_w + ox) * channels; + acc[0] += ws[k] * px[0]; + acc[1] += ws[k] * px[1]; + acc[2] += ws[k] * px[2]; + } + for (size_t c = 0; c < channels; ++c) { + const double normalized = (acc[c] / 255.0 - image_mean[c]) / image_std[c]; + chw[(c * target_h + oy) * target_w + ox] = static_cast(normalized); + } + } + } + return chw; +} + +// Convert a normalized CHW float image into flattened patches with the SigLIP2 +// merge-block ordering, producing a [num_patches_h * num_patches_w, C * patch_size^2] tensor. +// Mirrors image_processing_siglip2_fast.convert_image_to_patches with merge_size = 2. +ov::Tensor convert_image_to_patches(const std::vector& chw_image, + size_t channels, + size_t height, + size_t width, + size_t patch_size, + size_t merge_size) { + const size_t nph = height / patch_size; + const size_t npw = width / patch_size; + const size_t feat = channels * patch_size * patch_size; + const size_t num_patches = nph * npw; + + ov::Tensor patches(ov::element::f32, ov::Shape{num_patches, feat}); + float* dst = patches.data(); + const float* src = chw_image.data(); // CHW layout: [c][y][x] + + // Iterate patches in merge-block order: outer 2x2 block grid, inner 2x2 within block. + // Original numpy: reshape [C, nph/m, m, ps, npw/m, m, ps] -> permute(1,4,2,5,3,6,0) + // -> [nph/m, npw/m, m_h, m_w, ps_h, ps_w, C] -> reshape [num_patches, C*ps*ps]. + size_t patch_idx = 0; + for (size_t bh = 0; bh < nph / merge_size; ++bh) { + for (size_t bw = 0; bw < npw / merge_size; ++bw) { + for (size_t mh = 0; mh < merge_size; ++mh) { + for (size_t mw = 0; mw < merge_size; ++mw) { + const size_t patch_row = bh * merge_size + mh; + const size_t patch_col = bw * merge_size + mw; + float* row = dst + patch_idx * feat; + // Within-patch layout is [ps_h][ps_w][C] (row-major HWC). + size_t f = 0; + for (size_t py = 0; py < patch_size; ++py) { + for (size_t px = 0; px < patch_size; ++px) { + const size_t y = patch_row * patch_size + py; + const size_t x = patch_col * patch_size + px; + for (size_t c = 0; c < channels; ++c) { + row[f++] = src[(c * height + y) * width + x]; + } + } + } + ++patch_idx; + } + } + } + } + return patches; +} + +// Per-patch 2-D (h, w) rotary position embedding, honoring the spatial-merge block +// ordering, computed with theta = 10000. Output shape [num_patches, head_dim]. +// Mirrors _OVYoutuVLForCausalLM.rot_pos_emb. +ov::Tensor compute_rotary_pos_emb(size_t grid_h, size_t grid_w, size_t merge_size, size_t rope_dim) { + // rope_dim is the vision head_dim // 2, and the inv_freq has rope_dim // 2 entries. + const size_t half = rope_dim / 2; + std::vector inv_freq(half); + for (size_t i = 0; i < half; ++i) { + inv_freq[i] = 1.0f / std::pow(10000.0f, static_cast(2 * i) / static_cast(rope_dim)); + } + + // hpos_ids / wpos_ids in merge-block order (see rot_pos_emb). + const size_t num_patches = grid_h * grid_w; + std::vector hpos(num_patches); + std::vector wpos(num_patches); + size_t idx = 0; + for (size_t bh = 0; bh < grid_h / merge_size; ++bh) { + for (size_t bw = 0; bw < grid_w / merge_size; ++bw) { + for (size_t mh = 0; mh < merge_size; ++mh) { + for (size_t mw = 0; mw < merge_size; ++mw) { + hpos[idx] = bh * merge_size + mh; + wpos[idx] = bw * merge_size + mw; + ++idx; + } + } + } + } + + ov::Tensor rotary(ov::element::f32, ov::Shape{num_patches, rope_dim}); + float* out = rotary.data(); + for (size_t p = 0; p < num_patches; ++p) { + float* row = out + p * rope_dim; + // First half from height position, second half from width position (flatten(1) of stack). + for (size_t i = 0; i < half; ++i) { + row[i] = static_cast(hpos[p]) * inv_freq[i]; + } + for (size_t i = 0; i < half; ++i) { + row[half + i] = static_cast(wpos[p]) * inv_freq[i]; + } + } + return rotary; +} + +// Window index and cumulative window sequence lengths for the SigLIP2 windowed +// attention. Mirrors _OVYoutuVLForCausalLM.get_window_index. +std::pair, std::vector> compute_window_index(size_t grid_h, + size_t grid_w, + size_t merge_size, + size_t patch_size, + size_t window_size) { + const size_t spatial_merge_unit = merge_size * merge_size; + const size_t vit_merger_window_size = window_size / merge_size / patch_size; + + const size_t llm_grid_h = grid_h / merge_size; + const size_t llm_grid_w = grid_w / merge_size; + + const size_t pad_h = (vit_merger_window_size - llm_grid_h % vit_merger_window_size) % vit_merger_window_size; + const size_t pad_w = (vit_merger_window_size - llm_grid_w % vit_merger_window_size) % vit_merger_window_size; + const size_t num_windows_h = (llm_grid_h + pad_h) / vit_merger_window_size; + const size_t num_windows_w = (llm_grid_w + pad_w) / vit_merger_window_size; + + std::vector window_index; + window_index.reserve(llm_grid_h * llm_grid_w); + std::vector cu_window_seqlens; + cu_window_seqlens.push_back(0); + + for (size_t wh = 0; wh < num_windows_h; ++wh) { + for (size_t ww = 0; ww < num_windows_w; ++ww) { + size_t seqlen = 0; + for (size_t ih = 0; ih < vit_merger_window_size; ++ih) { + for (size_t iw = 0; iw < vit_merger_window_size; ++iw) { + const size_t gh = wh * vit_merger_window_size + ih; + const size_t gw = ww * vit_merger_window_size + iw; + if (gh < llm_grid_h && gw < llm_grid_w) { + window_index.push_back(static_cast(gh * llm_grid_w + gw)); + ++seqlen; + } + } + } + const int32_t prev = cu_window_seqlens.back(); + cu_window_seqlens.push_back(prev + static_cast(seqlen * spatial_merge_unit)); + } + } + + // torch.unique_consecutive on cu_window_seqlens. + std::vector unique_cws; + for (int32_t v : cu_window_seqlens) { + if (unique_cws.empty() || unique_cws.back() != v) { + unique_cws.push_back(v); + } + } + return {std::move(window_index), std::move(unique_cws)}; +} + +// Block-diagonal float attention mask: 0 inside a block, -inf outside. +ov::Tensor make_block_diag_mask(size_t seq_len, const std::vector& cu_seqlens) { + ov::Tensor mask(ov::element::f32, ov::Shape{1, seq_len, seq_len}); + float* data = mask.data(); + const float neg_inf = -std::numeric_limits::infinity(); + std::fill(data, data + seq_len * seq_len, neg_inf); + for (size_t b = 1; b < cu_seqlens.size(); ++b) { + const size_t start = static_cast(cu_seqlens[b - 1]); + const size_t end = static_cast(cu_seqlens[b]); + for (size_t i = start; i < end; ++i) { + for (size_t j = start; j < end; ++j) { + data[i * seq_len + j] = 0.0f; + } + } + } + return mask; +} + +} // namespace + +EncodedImage VisionEncoderYoutuVL::encode(const ov::Tensor& image, const ov::AnyMap& config_map) { + CircularBufferQueueElementGuard infer_request_guard(this->m_ireq_queue_vision_encoder.get()); + ov::InferRequest& encoder = infer_request_guard.get(); + ProcessorConfig config = ProcessorConfig::from_any_map(config_map, m_processor_config); + + const size_t patch_size = config.patch_size; + const size_t merge_size = 2; // spatial_merge_size for Youtu-VL / SigLIP2 + // YoutuVLProcessor.__call__ forwards max_image_patches=36864 to the SigLIP2 + // fast image processor (its own class attribute default of 256 is overridden + // by the processor), so images are kept at near-native resolution and only + // downscaled when they exceed 36864 patches. Using the un-overridden 256 here + // aggressively downscales large images, yielding far fewer vision tokens and + // materially worse answers, so the processor-level value must be used. + const size_t max_num_patches = 36864; // YoutuVLProcessor default max_image_patches + const size_t window_size = 256; // patch_size * 2 * 8 + + // 1. Convert input tensor (NHWC uint8) to clip_image_u8 (HWC uint8). + clip_image_u8 input_image = tensor_to_clip_image_u8(image); + const size_t orig_h = static_cast(input_image.ny); + const size_t orig_w = static_cast(input_image.nx); + + // 2. Resize to a patch-aligned target size, then rescale (1/255) + normalize. + // The SigLIP2 fast image processor resizes with anti-aliased bilinear + // interpolation (torchvision resize, antialias=True). Using a plain + // (non-anti-aliased) bilinear resize here produces visibly different pixel + // values on large downscales and shifts the vision features enough to change + // fine-grained answers, so the anti-aliased path is required to match + // optimum-intel. The helper returns the normalized image directly in CHW. + auto [target_h, target_w] = get_target_image_size(orig_h, orig_w, patch_size, max_num_patches); + std::vector normalized_chw = antialias_resize_normalize_chw( + input_image, target_h, target_w, config.image_mean.data(), config.image_std.data()); + + const size_t grid_h = target_h / patch_size; + const size_t grid_w = target_w / patch_size; + + // 3. Patchify into [num_patches, 3 * patch_size^2] in merge-block order. + ov::Tensor pixel_values = convert_image_to_patches(normalized_chw, 3, target_h, target_w, patch_size, merge_size); + const size_t seq_len = grid_h * grid_w; + + // 4. Recompute the auxiliary tensors expected by the fused vision model. + // rope_dim is inferred from the model's rotary_pos_emb input (last dim). + size_t rope_dim = 0; + for (const auto& port : encoder.get_compiled_model().inputs()) { + if (port.get_any_name() == "rotary_pos_emb") { + const auto& pshape = port.get_partial_shape(); + OPENVINO_ASSERT(pshape.rank().is_static() && pshape.size() == 2 && pshape[1].is_static(), + "Youtu-VL vision model rotary_pos_emb must have a static last dimension."); + rope_dim = static_cast(pshape[1].get_length()); + } + } + OPENVINO_ASSERT(rope_dim > 0, "Youtu-VL vision model is missing the 'rotary_pos_emb' input."); + ov::Tensor rotary_pos_emb = compute_rotary_pos_emb(grid_h, grid_w, merge_size, rope_dim); + + auto [window_index_vec, cu_window_seqlens] = compute_window_index(grid_h, grid_w, merge_size, patch_size, window_size); + + // cu_seqlens: cumulative patch counts per image (single image here). + std::vector cu_seqlens = {0, static_cast(seq_len)}; + + ov::Tensor attention_mask = make_block_diag_mask(seq_len, cu_seqlens); + ov::Tensor window_attention_mask = make_block_diag_mask(seq_len, cu_window_seqlens); + + ov::Tensor window_index(ov::element::i64, ov::Shape{window_index_vec.size()}); + std::copy(window_index_vec.begin(), window_index_vec.end(), window_index.data()); + + // 5. Run the fused vision tower + merger. + encoder.set_tensor("pixel_values", pixel_values); + encoder.set_tensor("attention_mask", attention_mask); + encoder.set_tensor("window_attention_mask", window_attention_mask); + encoder.set_tensor("window_index", window_index); + encoder.set_tensor("rotary_pos_emb", rotary_pos_emb); + encoder.infer(); + + const ov::Tensor& infer_output = encoder.get_output_tensor(); + ov::Tensor image_features(infer_output.get_element_type(), infer_output.get_shape()); + std::memcpy(image_features.data(), infer_output.data(), infer_output.get_byte_size()); + + EncodedImage encoded_image; + encoded_image.resized_source = std::move(image_features); + encoded_image.resized_source_size = ImageSize{grid_h, grid_w}; + encoded_image.original_image_size = ImageSize{orig_h, orig_w}; + return encoded_image; +} + +InputsEmbedderYoutuVL::InputsEmbedderYoutuVL( + const VLMConfig& vlm_config, + const std::filesystem::path& model_dir, + const Tokenizer& tokenizer, + const std::string& device, + const ov::AnyMap device_config) : + IInputsEmbedder(vlm_config, model_dir, tokenizer, device, device_config) { + encode_vision_placeholder_tokens(); +} + +InputsEmbedderYoutuVL::InputsEmbedderYoutuVL( + const VLMConfig& vlm_config, + const ModelsMap& models_map, + const Tokenizer& tokenizer, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config) : + IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) { + encode_vision_placeholder_tokens(); +} + +void InputsEmbedderYoutuVL::encode_vision_placeholder_tokens() { + auto encoded_vision_tokens = m_tokenizer.encode(m_vlm_config.vision_start_token + + m_vlm_config.vision_end_token + + m_vlm_config.image_pad_token, + ov::genai::add_special_tokens(false)); + const int64_t* ids = encoded_vision_tokens.input_ids.data(); + OPENVINO_ASSERT(encoded_vision_tokens.input_ids.get_size() >= 3, + "Failed to encode Youtu-VL vision placeholder tokens."); + m_vision_token_ids["vision_start"] = ids[0]; + m_vision_token_ids["vision_end"] = ids[1]; + m_vision_token_ids["image_pad"] = ids[2]; +} + +size_t InputsEmbedderYoutuVL::calc_tokens_num(size_t grid_h, size_t grid_w) const { + return grid_h * grid_w / m_merge_length; +} + +NormalizedPrompt InputsEmbedderYoutuVL::normalize_prompt(const std::string& prompt, + size_t base_id, + const std::vector& images) const { + auto [unified_prompt, images_sequence] = + normalize(prompt, NATIVE_TAG, NATIVE_TAG, base_id, images.size(), VisionType::IMAGE); + + for (size_t new_image_id : images_sequence) { + const auto& encoded_image = images.at(new_image_id - base_id); + const size_t grid_h = encoded_image.resized_source_size.height; + const size_t grid_w = encoded_image.resized_source_size.width; + const size_t num_image_pad_tokens = calc_tokens_num(grid_h, grid_w); + + std::string expanded_tag; + expanded_tag.reserve(m_vlm_config.vision_start_token.length() + + m_vlm_config.image_pad_token.length() * num_image_pad_tokens + + m_vlm_config.vision_end_token.length()); + expanded_tag.append(m_vlm_config.vision_start_token); + for (size_t i = 0; i < num_image_pad_tokens; ++i) { + expanded_tag.append(m_vlm_config.image_pad_token); + } + expanded_tag.append(m_vlm_config.vision_end_token); + + const auto pos = unified_prompt.find(NATIVE_TAG); + OPENVINO_ASSERT(pos != std::string::npos, "Failed to locate Youtu-VL image tag in prompt."); + unified_prompt.replace(pos, NATIVE_TAG.length(), expanded_tag); + } + + return {std::move(unified_prompt), std::move(images_sequence), {}}; +} + +ov::Tensor InputsEmbedderYoutuVL::get_inputs_embeds(const std::string& unified_prompt, + const std::vector& images, + ov::genai::VLMPerfMetrics& metrics, + bool recalculate_merged_embeddings, + const std::vector& images_sequence) { + ov::Tensor input_ids = get_encoded_input_ids(unified_prompt, metrics); + CircularBufferQueueElementGuard embeddings_request_guard(m_embedding->get_request_queue().get()); + EmbeddingsRequest& req = embeddings_request_guard.get(); + ov::Tensor text_embeds = m_embedding->infer(req, input_ids); + + if (images.empty()) { + ov::Tensor inputs_embeds(text_embeds.get_element_type(), text_embeds.get_shape()); + std::memcpy(inputs_embeds.data(), text_embeds.data(), text_embeds.get_byte_size()); + return inputs_embeds; + } + + const int64_t image_pad_token_id = m_vision_token_ids.at("image_pad"); + + // Merge image embeddings into text embeddings at <|image_pad|> token positions, + // consuming per-image embedding rows in prompt order. + const ov::Shape text_shape = text_embeds.get_shape(); // [1, seq_len, hidden] + const size_t seq_len = text_shape[1]; + const size_t hidden_size = text_shape[2]; + + ov::Tensor merged_embeds(text_embeds.get_element_type(), text_shape); + std::memcpy(merged_embeds.data(), text_embeds.data(), text_embeds.get_byte_size()); + + const int64_t* input_ids_data = input_ids.data(); + float* merged_data = merged_embeds.data(); + + // Order the image embeddings by their appearance in the prompt. + std::vector image_embed_ptrs; + std::vector image_embed_rows; + size_t total_image_rows = 0; + for (size_t new_image_id : images_sequence) { + const ov::Tensor& src = images.at(new_image_id).resized_source; + const size_t rows = src.get_shape().at(0); + image_embed_ptrs.push_back(src.data()); + image_embed_rows.push_back(rows); + total_image_rows += rows; + } + + size_t cur_image = 0; + size_t cur_row = 0; + size_t consumed_rows = 0; + for (size_t s = 0; s < seq_len; ++s) { + if (input_ids_data[s] == image_pad_token_id) { + OPENVINO_ASSERT(cur_image < image_embed_ptrs.size(), + "Youtu-VL: more <|image_pad|> tokens than available image embeddings."); + const float* src_row = image_embed_ptrs[cur_image] + cur_row * hidden_size; + std::copy_n(src_row, hidden_size, merged_data + s * hidden_size); + ++cur_row; + ++consumed_rows; + if (cur_row == image_embed_rows[cur_image]) { + ++cur_image; + cur_row = 0; + } + } + } + OPENVINO_ASSERT(consumed_rows == total_image_rows, + "Youtu-VL: number of <|image_pad|> tokens (", consumed_rows, + ") does not match total image embedding rows (", total_image_rows, ")."); + + return merged_embeds; +} + +} // namespace ov::genai diff --git a/src/cpp/src/visual_language/youtu_vl/classes.hpp b/src/cpp/src/visual_language/youtu_vl/classes.hpp new file mode 100644 index 0000000000..6f708b92ae --- /dev/null +++ b/src/cpp/src/visual_language/youtu_vl/classes.hpp @@ -0,0 +1,73 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "visual_language/vlm_config.hpp" + +#include "visual_language/vision_encoder.hpp" +#include "visual_language/inputs_embedder.hpp" +#include "circular_buffer_queue.hpp" + +namespace ov::genai { + +// Vision encoder for tencent/Youtu-VL-4B-Instruct. +// The SigLIP2 windowed vision tower and the VLPatchMerger are exported as a single +// `openvino_vision_embeddings_model` that consumes materialized auxiliary tensors +// (attention_mask, window_attention_mask, window_index, rotary_pos_emb). These are +// recomputed here from the image spatial shape, mirroring optimum-intel's +// _OVYoutuVLForCausalLM runtime wrapper (see modeling_visual_language.py). +class VisionEncoderYoutuVL : public VisionEncoder { +public: + using VisionEncoder::VisionEncoder; + + EncodedImage encode(const ov::Tensor& image, const ov::AnyMap& config_map) override; +}; + +class InputsEmbedderYoutuVL : public InputsEmbedder::IInputsEmbedder { +public: + InputsEmbedderYoutuVL( + const VLMConfig& vlm_config, + const std::filesystem::path& model_dir, + const Tokenizer& tokenizer, + const std::string& device, + const ov::AnyMap device_config); + + InputsEmbedderYoutuVL( + const VLMConfig& vlm_config, + const ModelsMap& models_map, + const Tokenizer& tokenizer, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config); + + ov::Tensor get_inputs_embeds(const std::string& prompt, + const std::vector& images, + ov::genai::VLMPerfMetrics& metrics, + bool recalculate_merged_embeddings = true, + const std::vector& image_sequence = {}) override; + + NormalizedPrompt normalize_prompt( + const std::string& prompt, + size_t base_id, + const std::vector& images) const override; + + // Youtu-VL text backbone uses plain 1-D position ids (base implementation), so + // get_position_ids / get_generation_phase_position_ids are NOT overridden. + +protected: + // The chat template hardcodes the vision tag sequence, so NATIVE_TAG is hardcoded too. + inline static const std::string NATIVE_TAG = "<|vision_start|><|image_pad|><|vision_end|>"; + + void encode_vision_placeholder_tokens(); + + // Number of merged image tokens for a (grid_h, grid_w) patch grid. + size_t calc_tokens_num(size_t grid_h, size_t grid_w) const; + + std::map m_vision_token_ids; + size_t m_merge_length = 4; // spatial_merge_size ** 2 +}; + +} // namespace ov::genai diff --git a/tests/python_tests/test_vlm_pipeline.py b/tests/python_tests/test_vlm_pipeline.py index 9ebee38bac..0814750433 100644 --- a/tests/python_tests/test_vlm_pipeline.py +++ b/tests/python_tests/test_vlm_pipeline.py @@ -32,6 +32,7 @@ patch_pyav_for_servercore.install_av_stub_module_for_windows() import inspect +import functools from enum import Enum from dataclasses import dataclass from pathlib import Path @@ -101,6 +102,30 @@ def _is_videochat_flash_qwen_model(model_id: str) -> bool: return "videochat-flash-qwen" in model_id.lower() +@functools.lru_cache(maxsize=None) +def _hub_model_available(model_id: str) -> bool: + """Return True when the given model id is resolvable on the HuggingFace Hub.""" + from huggingface_hub import HfApi + from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError + + try: + HfApi().model_info(model_id) + return True + except (RepositoryNotFoundError, HfHubHTTPError): + return False + except Exception: + return False + + +def _youtu_vl_fixture_available() -> bool: + """The tiny-random youtu_vl fixture is usable if it is either published on the + Hub or already present as a converted OpenVINO IR in the local OV_CACHE.""" + converted_dir = get_ov_cache_converted_models_dir() / str(MODEL_YOUTU_VL).replace(os.sep, "_") + if (converted_dir / "openvino_language_model.xml").exists(): + return True + return _hub_model_available(MODEL_YOUTU_VL) + + VIDEOCHAT_FLASH_QWEN_MODEL_ID = "optimum-intel-internal-testing/tiny-videochat-flash-qwen" @@ -167,6 +192,9 @@ def __getattr__(self, name: str): MODEL_GEMMA = "optimum-intel-internal-testing/tiny-random-gemma3" MODEL_GEMMA3N = "optimum-intel-internal-testing/tiny-random-gemma3n" MODEL_QWEN3_OMNI = "optimum-intel-internal-testing/tiny-random-qwen3-omni" +# tencent/Youtu-VL-4B-Instruct: SigLIP2 windowed vision tower + DeepSeek-V3-style MLA/MoE +# text backbone. Uses Qwen-style vision tags but plain 1-D LM position ids. +MODEL_YOUTU_VL = "optimum-intel-internal-testing/tiny-random-youtu-vl" MODEL_IDS: list[str] = [] if is_transformers_version("<", "5.0"): @@ -181,6 +209,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-gemma3", MODEL_GEMMA3N, "optimum-intel-internal-testing/tiny-random-MiniCPM-o-2_6", + MODEL_YOUTU_VL, *VIDEO_MODEL_IDS, ] else: @@ -188,6 +217,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-phi3-vision", "optimum-intel-internal-testing/tiny-random-phi-4-multimodal", "qnguyen3/nanoLLaVA", + MODEL_YOUTU_VL, *VIDEO_MODEL_IDS, ] @@ -218,6 +248,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-gemma4-31B": lambda idx: "<|image|>", "qnguyen3/nanoLLaVA": lambda idx: "\n", VIDEOCHAT_FLASH_QWEN_MODEL_ID: lambda idx: f"<|image_{idx + 1}|>\n", + MODEL_YOUTU_VL: lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", } @@ -244,6 +275,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-qwen2.5-vl": 336, "optimum-intel-internal-testing/tiny-random-qwen3-vl": 256, "optimum-intel-internal-testing/tiny-random-qwen3.5": 256, + MODEL_YOUTU_VL: 64, } @@ -345,6 +377,17 @@ def _maybe_skip_unsupported_model_export(model_id: str) -> None: if _is_videochat_flash_qwen_model(model_id) and not is_optimum_intel_version_for_videochat_flash_qwen(): pytest.skip("ValueError: The current version of optimum-intel does not support videochat_flash_qwen") + if model_id == MODEL_YOUTU_VL and not _youtu_vl_fixture_available(): + # The tiny-random youtu_vl fixture is not published on the Hub yet. + # youtu_vl support is validated in GenAI against the real + # tencent/Youtu-VL-4B-Instruct export; this repository test activates once + # the tiny-random fixture is hosted (or provided via the local OV_CACHE + # converted-models cache). See CVS ticket for fixture publication. + pytest.skip( + f"Tiny-random youtu_vl fixture '{MODEL_YOUTU_VL}' is not published on the " + "HuggingFace Hub yet." + ) + def _get_vlm_eagle3_model_paths() -> tuple[Path, Path]: _maybe_skip_unsupported_model_export(VLM_EAGLE3_MAIN_MODEL_ID) @@ -432,6 +475,7 @@ def convert_to_temp(temp_dir: Path) -> None: "qnguyen3/nanoLLaVA", "optimum-intel-internal-testing/tiny-random-MiniCPM-o-2_6", VIDEOCHAT_FLASH_QWEN_MODEL_ID, + MODEL_YOUTU_VL, }, ) ) diff --git a/tools/who_what_benchmark/tests/test_unit_youtu_vl.py b/tools/who_what_benchmark/tests/test_unit_youtu_vl.py new file mode 100644 index 0000000000..d051ea3b4b --- /dev/null +++ b/tools/who_what_benchmark/tests/test_unit_youtu_vl.py @@ -0,0 +1,94 @@ +# Copyright (C) 2023-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from PIL import Image + + +def test_youtu_vl_preprocessor_registered(): + """youtu_vl must resolve to a VLM inputs preprocessor so the HF visual-text + path does not fall through to the optimum-intel mapping (which would raise + KeyError('youtu_vl')).""" + from whowhatbench.inputs_preprocessors import ( + MODEL_TYPE_TO_CLS_MAPPING, + YoutuVLInputsPreprocessor, + VLMInputsPreprocessor, + ) + + assert "youtu_vl" in MODEL_TYPE_TO_CLS_MAPPING + cls = MODEL_TYPE_TO_CLS_MAPPING["youtu_vl"] + assert cls is YoutuVLInputsPreprocessor + assert issubclass(cls, VLMInputsPreprocessor) + + +def test_youtu_vl_preprocessor_builds_multimodal_inputs(): + """The youtu_vl preprocessor should render the chat template and forward the + image to the processor (Qwen2-VL style), without any Hub-id special casing.""" + from whowhatbench.inputs_preprocessors import YoutuVLInputsPreprocessor + + captured = {} + + class FakeProcessor: + def apply_chat_template(self, conversation, add_generation_prompt, tokenize): + captured["conversation"] = conversation + return "rendered-prompt" + + def __call__(self, images, text, videos, return_tensors): + captured["images"] = images + captured["text"] = text + captured["videos"] = videos + return {"input_ids": [[1, 2, 3]]} + + img = Image.new("RGB", (8, 8), color=(10, 20, 30)) + pre = YoutuVLInputsPreprocessor() + out = pre.preprocess_inputs("What is this?", image=img, processor=FakeProcessor()) + + assert out == {"input_ids": [[1, 2, 3]]} + assert captured["text"] == "rendered-prompt" + assert captured["images"] is img + assert captured["videos"] is None + # user message must carry both the image and the text content. + content_types = [c["type"] for c in captured["conversation"][0]["content"]] + assert "image" in content_types and "text" in content_types + + +def test_load_prompts_local_csv_visual_text(tmp_path): + """--dataset pointing at a local CSV with prompts/images/videos should be + loaded generically, resolving image paths relative to the CSV directory.""" + from whowhatbench import wwb + + img_dir = tmp_path / "images" + img_dir.mkdir() + img_path = img_dir / "a.png" + Image.new("RGB", (8, 8), color=(1, 2, 3)).save(img_path) + + csv_path = tmp_path / "inputs.csv" + csv_path.write_text( + "prompts,images,videos\n" + "Describe this,images/a.png,\n" + ) + + args = SimpleNamespace(dataset=str(csv_path), model_type="visual-text", + split=None, dataset_field="prompts") + res = wwb.load_prompts(args) + + assert res["prompts"] == ["Describe this"] + assert isinstance(res["images"][0], Image.Image) + assert res["images"][0].mode == "RGB" + assert res["videos"] == [None] + + +def test_load_prompts_local_csv_requires_prompts_column(tmp_path): + from whowhatbench import wwb + + csv_path = tmp_path / "bad.csv" + csv_path.write_text("questions,images\nhi,x.png\n") + args = SimpleNamespace(dataset=str(csv_path), model_type="visual-text", + split=None, dataset_field="prompts") + try: + wwb.load_prompts(args) + except ValueError as e: + assert "prompts" in str(e) + else: + raise AssertionError("expected ValueError for missing 'prompts' column") diff --git a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py index 453ed6f025..e09ce8493b 100644 --- a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py +++ b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py @@ -8,6 +8,7 @@ from .qwen3 import Qwen3VLInputsPreprocessor, Qwen3_5VLInputsPreprocessor, Qwen3OmniInputsPreprocessor from .gemma3 import Gemma3InputsPreprocessor from .gemma4 import Gemma4InputsPreprocessor, Gemma4UnifiedInputsPreprocessor, Gemma3nInputsPreprocessor +from .youtu import YoutuVLInputsPreprocessor from .vlm_inputs_preprocessor import VLMInputsPreprocessor MODEL_TYPE_TO_CLS_MAPPING = { @@ -33,6 +34,7 @@ "llava_next": LLAVAInputsPreprocessor, "llava-qwen2": NanoLlavaInputsPreprocessor, "internvl_chat": InternVLInputsPreprocessor, + "youtu_vl": YoutuVLInputsPreprocessor, } __all__ = ["MODEL_TYPE_TO_CLS_MAPPING", "VLMInputsPreprocessor"] diff --git a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/youtu.py b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/youtu.py new file mode 100644 index 0000000000..678c685c02 --- /dev/null +++ b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/youtu.py @@ -0,0 +1,90 @@ +import numpy as np +from transformers import ( + AutoImageProcessor, + PretrainedConfig, + PreTrainedTokenizer, +) +from .vlm_inputs_preprocessor import VLMInputsPreprocessor +from typing import TYPE_CHECKING, Optional, Union, Any + +if TYPE_CHECKING: + from PIL.Image import Image + from transformers.image_utils import VideoInput + + +class YoutuVLInputsPreprocessor(VLMInputsPreprocessor): + """Inputs preprocessor for tencent Youtu-VL models (model_type="youtu_vl"). + + Youtu-VL reuses the Qwen2-VL style multimodal chat interface: media + placeholders are expressed via <|vision_start|><|image_pad|><|vision_end|> + in the chat template, and the processor consumes the rendered text together + with the raw images/videos. The preprocessing is therefore identical in + structure to Qwen2VLInputsPreprocessor and is kept generic (no Hub-id + special casing) so it applies to the whole youtu_vl architecture. + """ + + def __init__(self, chat_mode: bool = False, model: Optional[Any] = None): + super().__init__(chat_mode) + if model is not None: + self.def_image_token_id = getattr(model.config, "image_token_id", None) + + def update_chat_history_with_answer(self, answer): + self.chat_history.append({"role": "assistant", "content": [{"type": "text", "text": answer}]}) + + def preprocess_inputs( + self, + text: str, + image: Optional[Union["Image", list["Image"]]] = None, + processor: Optional[AutoImageProcessor] = None, + tokenizer: Optional[PreTrainedTokenizer] = None, + config: Optional[PretrainedConfig] = None, + video: Optional[Union["VideoInput", list["VideoInput"]]] = None, + audio: Optional[np.ndarray] = None, + ): + if processor is None: + raise ValueError("Processor is required.") + if audio is not None: + raise ValueError("Audio input is not supported") + + self.update_images(image) + media = [] + if image is not None: + if not isinstance(image, list): + image = [image] + media += [{"type": "image", "image": img} for img in image] + + if video is not None: + if not isinstance(video, list): + video = [video] + media += [{"type": "video", "video": v} for v in video] + + if self.chat_mode: + if self.videos is None: + self.videos = [] + self.videos.extend(video) + else: + self.videos = video + elif not self.chat_mode: + self.videos = None + + new_message = {"role": "user", "content": media + [{"type": "text", "text": text}]} + if self.chat_mode: + self.chat_history.append(new_message) + conversation = self.chat_history + else: + conversation = [new_message] + + text_prompt = processor.apply_chat_template( + conversation, + add_generation_prompt=True, + tokenize=False, + ) + + inputs = processor( + images=self.images, + text=text_prompt, + videos=self.videos, + return_tensors="pt", + ) + + return inputs diff --git a/tools/who_what_benchmark/whowhatbench/model_loaders.py b/tools/who_what_benchmark/whowhatbench/model_loaders.py index 7d4cfb24a2..5359ffedc4 100644 --- a/tools/who_what_benchmark/whowhatbench/model_loaders.py +++ b/tools/who_what_benchmark/whowhatbench/model_loaders.py @@ -570,7 +570,12 @@ def load_visual_text_model( config._attn_implementation = "sdpa" from_pretrained_kwargs = {"config": config} else: - from_pretrained_kwargs = {"_attn_implementation": "eager", "use_flash_attention_2": False} + # `_attn_implementation="eager"` already disables flash attention. + # The legacy `use_flash_attention_2` flag is not accepted by every + # model __init__ (e.g. custom remote-code architectures like + # youtu_vl, whose signature is (self, config)) and forwarding it raises + # TypeError. Keep only the portable attention selector here. + from_pretrained_kwargs = {"_attn_implementation": "eager"} model = AutoModelForCausalLM.from_pretrained( model_id, diff --git a/tools/who_what_benchmark/whowhatbench/wwb.py b/tools/who_what_benchmark/whowhatbench/wwb.py index 1c0c80c04f..769a0778d0 100644 --- a/tools/who_what_benchmark/whowhatbench/wwb.py +++ b/tools/who_what_benchmark/whowhatbench/wwb.py @@ -518,9 +518,64 @@ def check_args(args): raise ValueError("--llamacpp-chat requires --llamacpp") +def _load_local_csv_prompts(csv_path, model_type): + """Load prompts from a local CSV file through the --dataset interface. + + The CSV must contain a "prompts" column. For visual-text style tasks it may + also contain "images" and "videos" columns holding file paths (or empty + cells). Media paths are resolved deterministically relative to the CSV's + directory and images are eagerly loaded into RGB PIL objects so the + downstream evaluator receives ready-to-use inputs. This keeps the remote + dataset loader optional (e.g. when it is unavailable or too slow) without + any model-name special casing. + """ + df = pd.read_csv(csv_path, keep_default_na=False) + if "prompts" not in df.columns: + raise ValueError(f"Local dataset CSV '{csv_path}' must contain a 'prompts' column.") + + base_dir = os.path.dirname(os.path.abspath(csv_path)) + + def _resolve(path): + path = str(path).strip() + if not path: + return None + return path if os.path.isabs(path) else os.path.join(base_dir, path) + + res = {"prompts": [str(p) for p in df["prompts"].tolist()]} + + is_visual = model_type in ("visual-text", "visual-text-chat", "visual-video-text") + if is_visual or "images" in df.columns or "videos" in df.columns: + images = [] + if "images" in df.columns: + for cell in df["images"].tolist(): + resolved = _resolve(cell) + images.append(Image.open(resolved).convert("RGB") if resolved else None) + else: + images = [None] * len(res["prompts"]) + + videos = [] + if "videos" in df.columns: + for cell in df["videos"].tolist(): + videos.append(_resolve(cell)) + else: + videos = [None] * len(res["prompts"]) + + res["images"] = images + res["videos"] = videos + + return res + + def load_prompts(args): if args.dataset is None: return None + + # Allow a local CSV to be supplied through the same --dataset interface. + # This is the generic escape hatch when the default remote dataset loader + # is unavailable or too slow (notably for visual-text ground-truth runs). + if os.path.isfile(args.dataset) and args.dataset.lower().endswith(".csv"): + return _load_local_csv_prompts(args.dataset, args.model_type) + split = "validation" if args.split is not None: split = args.split