diff --git a/.model_analysis/inspect_ir.py b/.model_analysis/inspect_ir.py new file mode 100644 index 0000000000..dd0bda3b31 --- /dev/null +++ b/.model_analysis/inspect_ir.py @@ -0,0 +1,15 @@ +from openvino import Core +from pathlib import Path +import sys + +core = Core() +for xml in sorted(Path(sys.argv[1]).glob("openvino_*.xml")): + if "tokenizer" in xml.name or "detokenizer" in xml.name: + print(f"\n=== {xml.name} === SKIPPED (tokenizer)") + continue + m = core.read_model(xml) + print(f"\n=== {xml.name} ===") + for i in m.inputs: + print(f" IN {i.any_name}: {i.partial_shape} {i.element_type}") + for o in m.outputs: + print(f" OUT {o.any_name}: {o.partial_shape} {o.element_type}") diff --git a/.model_analysis/jvlm_analysis.md b/.model_analysis/jvlm_analysis.md new file mode 100644 index 0000000000..90d864eba6 --- /dev/null +++ b/.model_analysis/jvlm_analysis.md @@ -0,0 +1,87 @@ +# Model Analysis: jinaai/jina-vlm (jvlm) + +## Identity +- model_id: /home/openvino_bot/.../workspace/tiny_jina_vlm (tiny-random of jinaai/jina-vlm) +- model_type: jvlm +- architecture: JinaVLMForConditionalGeneration (registered via remote code under AutoModelForCausalLM) +- task / modality: image-text-to-text (vision-text) +- transformers version: 4.57.6 optimum-intel version: 2.1.0.dev0+fd94990 +- Remote code: configuration_jvlm.py, modeling_jvlm.py, processing_jvlm.py, image_processing_jvlm.py, blocks_jvlm.py + +## Exported IR (5 submodels) +| File | Role | Inputs (name: shape, dtype) | Outputs | +|------|------|-----------------------------|---------| +| openvino_text_embeddings_model.xml | text token -> embedding | input: [?,?] i64 | inputs_embeds: [?,?,64] f32 | +| openvino_vision_embeddings_model.xml | vision encoder + connector | image_patches: [?,?,?,588] f32; image_masks: [?,?,?] i64 | last_hidden_state: [?,?,64] f32 | +| openvino_language_model.xml | LM (decoder, with past) | inputs_embeds: [?,?,64] f32; attention_mask: [?,?] i64; position_ids: [?,?] i64; beam_idx: [?] i32 | logits: [?,?,152064] f32 | +| openvino_tokenizer.xml | tokenizer | - | - | +| openvino_detokenizer.xml | detokenizer | - | - | + +Notes: +- vision input `image_patches` last dim 588 = patch_size(14)^2 * 3 channels = 196*3. +- LM uses standard inputs_embeds + attention_mask + position_ids + beam_idx (Qwen-style decoder). hidden_size=64 (tiny). Real model hidden_size larger. +- text_config.vocab_size=152064; additional_vocab_size=128 (special image tokens live in additional vocab). + +## Transformers (remote code) +- Processor: JinaVLMProcessor (processing_jvlm.py). Image special tokens: + - `` (patch_token_id, in-text placeholder for each image patch/token) + - `` / `` (image boundary) + - `` (column separator token) + - `<|image|>` (image prompt token, chat template placeholder) + - `` +- Image preprocessing: JinaVLMImageProcessor (image_processing_jvlm.py), Molmo-derived. + - cropping_method = "overlap-and-resize" (Molmo), max_crops=12, overlap_margins=[4,4] + - base_input_size=[378,378], patch_size=14, pooling 2x2, tokens_per_image=196 + - normalization: minmax (image_min=-1, image_max=1), image_mean/std = OPENAI_CLIP + - Produces: image_patches [n_crops, n_patches, 588], image_masks [n_crops, n_patches], image_input_idx [n_crops, n_tokens] +- Merge (modeling_jvlm / optimum-intel _OVJinaVLMForCausalLM.merge_vision_text_embeddings): + - Index-based scatter: inputs_embeds[batch_idx[valid], image_input_idx[valid]] = image_embeds[valid] + - image_input_idx gives absolute positions in the token sequence where each image embedding row is placed (>=0 valid, <0 skip). +- LM: Qwen-style RMSNorm decoder, RoPE theta 1e6, n_kv_heads=2, head_dim=16 (tiny). standard position_ids. + +## Optimum-Intel +- module path: optimum/intel/openvino/modeling_visual_language.py : class _OVJinaVLMForCausalLM (line 5646), registered key "jvlm" (line 7509). +- IR <-> logical mapping: + - vision_embeddings(image_patches, image_masks) -> last_hidden_state (already projected to hidden_size) + - text_embeddings(input_ids) -> inputs_embeds + - language_model(inputs_embeds, attention_mask, position_ids, beam_idx) -> logits +- get_vision_embeddings: skips vision when decoding a single new token (input_ids.shape[1]==1). +- merge_vision_text_embeddings: scatter by image_input_idx (see above). image_input_idx carried only on prefill step. +- preprocess_inputs: uses chat template + processor(images, text) -> {input_ids, image_patches, image_masks, image_input_idx}. + +## GenAI baseline +- openvino_genai.VLMPipeline(ov_model, "CPU") raises: "Unsupported 'jvlm' VLM model type" (vlm_config.cpp:43). jvlm not enabled in GenAI. + +## Notes / gaps for GenAI enablement +- No existing GenAI model uses image_input_idx scatter; needs a new merge strategy. +- Vision encoder IR consumes pre-patchified image_patches + image_masks (Molmo overlap-and-resize preprocessing must be reproduced in C++), plus image_input_idx computed on host. +- Special image tokens are in the additional vocab; the OV tokenizer must be checked to preserve their IDs. + +## GenAI Enablement Design +- Closest GenAI model: LLaVA — because jvlm merges vision embeddings into text at + placeholder-token positions. Validated (proto_merge.py) that scatter by + `image_input_idx` is IDENTICAL to sequential scatter onto `` (id 151938) + positions, so a simple placeholder scatter (llava-style, but per-token not per-block + because im_patch tokens are interrupted by im_col/im_start/im_end) reproduces the + optimum output exactly. +- Validated facts (Python prototypes under .model_enabler/jvlm/): + - OV tokenizer encodes special image tokens as single ids: =151936, + =151937, =151938, =151939, <|image|>=151940. + - Expanding `<|image|>` into the joint token string + [im_start (patch*TL + col)*TL im_end] per crop (global thumbnail first, then crops) + reproduces the processor input_ids EXACTLY (439 tokens for a 64x64 image). + - Vision IR output is [1, n_crops*196, hidden]; merge = fill positions in + order with these rows. + - Pure-numpy Molmo preprocessing (bilinear align_corners=False, minmax normalize) + reproduces image_masks exactly and image_patches within 2/255; LM tokens are + token-identical to the reference/optimum output. +- Required changes: + - vlm_config.hpp/.cpp: add VLMModelType::JVLM + "jvlm" mapping; add jvlm token strings. + - visual_language/jvlm/classes.{hpp,cpp}: VisionEncoderJVLM (Molmo preprocessing + + vision IR: inputs image_patches, image_masks) and InputsEmbedderJVLM + (normalize_prompt expands <|image|>, get_inputs_embeds scatters onto ). + - vision_encoder.cpp / inputs_embedder.cpp: factory registration; inputs_embedder.hpp friend. +- Gaps: no existing GenAI model uses image_input_idx or Molmo overlap-and-resize; both + implemented fresh. Vision IR takes pre-patchified image_patches [n_crops, n_patches, 588] + and image_masks [n_crops, n_patches]; EncodedImage carries these plus per-crop token + layout for prompt expansion. diff --git a/site/docs/supported-models/_components/vlm-models-table/models.ts b/site/docs/supported-models/_components/vlm-models-table/models.ts index acffe74603..b4bfcb5f9e 100644 --- a/site/docs/supported-models/_components/vlm-models-table/models.ts +++ b/site/docs/supported-models/_components/vlm-models-table/models.ts @@ -32,6 +32,15 @@ export const VLM_MODELS: VLMModelType[] = [ }, ], }, + { + architecture: 'JinaVLMForConditionalGeneration', + models: [ + { + name: 'Jina-VLM', + links: ['https://huggingface.co/jinaai/jina-vlm'], + }, + ], + }, { architecture: 'LLaVA', models: [ diff --git a/src/cpp/src/visual_language/inputs_embedder.cpp b/src/cpp/src/visual_language/inputs_embedder.cpp index 40660f5bcc..5d3f609dc8 100644 --- a/src/cpp/src/visual_language/inputs_embedder.cpp +++ b/src/cpp/src/visual_language/inputs_embedder.cpp @@ -25,6 +25,7 @@ #include "visual_language/gemma3n/classes.hpp" #include "visual_language/gemma4/classes.hpp" #include "visual_language/videochat_flash/classes.hpp" +#include "visual_language/jvlm/classes.hpp" #include "continuous_batching/timer.hpp" #include "utils.hpp" @@ -384,6 +385,8 @@ InputsEmbedder::InputsEmbedder(const std::filesystem::path& model_dir, m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); } else if (vlm_config.model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); + } else if (vlm_config.model_type == VLMModelType::JVLM) { + m_impl = std::make_shared(vlm_config, model_dir, tokenizer, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM InputsEmbedder class. Please, create feature request on new model support"); } @@ -432,6 +435,8 @@ InputsEmbedder::InputsEmbedder(const ModelsMap& models_map, m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); } else if (vlm_config.model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); + } else if (vlm_config.model_type == VLMModelType::JVLM) { + m_impl = std::make_shared(vlm_config, models_map, tokenizer, config_dir_path, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM InputsEmbedder class. Please, create feature request on new model support"); } diff --git a/src/cpp/src/visual_language/inputs_embedder.hpp b/src/cpp/src/visual_language/inputs_embedder.hpp index 2d3218dacf..217b394519 100644 --- a/src/cpp/src/visual_language/inputs_embedder.hpp +++ b/src/cpp/src/visual_language/inputs_embedder.hpp @@ -397,6 +397,7 @@ class InputsEmbedder { friend class InputsEmbedderGemma3n; friend class InputsEmbedderGemma4; friend class InputsEmbedderVideoChatFlashQwen; + friend class InputsEmbedderJVLM; }; template diff --git a/src/cpp/src/visual_language/jvlm/classes.cpp b/src/cpp/src/visual_language/jvlm/classes.cpp new file mode 100644 index 0000000000..c5b890544e --- /dev/null +++ b/src/cpp/src/visual_language/jvlm/classes.cpp @@ -0,0 +1,718 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "visual_language/jvlm/classes.hpp" + +#include +#include +#include +#include + +#include "visual_language/clip.hpp" +#include "json_utils.hpp" +#include "utils.hpp" + +namespace ov::genai { + +namespace { + +// PyTorch/torchvision bilinear resize with align_corners=False, antialias=False. +// Input is RGB uint8 (HWC). Output is float32 in [0, 1] (HWC), rounded to the +// nearest uint8 value first to match torchvision's uint8 resize semantics. +std::vector bilinear_resize_u8_to_unit(const clip_image_u8& img, + int out_h, + int out_w) { + const int in_h = img.ny; + const int in_w = img.nx; + std::vector out(static_cast(out_h) * out_w * 3); + const double scale_h = static_cast(in_h) / out_h; + const double scale_w = static_cast(in_w) / out_w; + for (int oy = 0; oy < out_h; ++oy) { + double src_y = (oy + 0.5) * scale_h - 0.5; + int y0 = static_cast(std::floor(src_y)); + double wy = src_y - y0; + int y0c = std::min(std::max(y0, 0), in_h - 1); + int y1c = std::min(std::max(y0 + 1, 0), in_h - 1); + for (int ox = 0; ox < out_w; ++ox) { + double src_x = (ox + 0.5) * scale_w - 0.5; + int x0 = static_cast(std::floor(src_x)); + double wx = src_x - x0; + int x0c = std::min(std::max(x0, 0), in_w - 1); + int x1c = std::min(std::max(x0 + 1, 0), in_w - 1); + for (int c = 0; c < 3; ++c) { + double Ia = img.buf[(static_cast(y0c) * in_w + x0c) * 3 + c]; + double Ib = img.buf[(static_cast(y0c) * in_w + x1c) * 3 + c]; + double Ic = img.buf[(static_cast(y1c) * in_w + x0c) * 3 + c]; + double Id = img.buf[(static_cast(y1c) * in_w + x1c) * 3 + c]; + double top = Ia * (1.0 - wx) + Ib * wx; + double bot = Ic * (1.0 - wx) + Id * wx; + double val = top * (1.0 - wy) + bot * wy; + // Match torchvision uint8 resize: round-half-to-even, clip, then /255. + double r = std::nearbyint(val); + if (r < 0.0) + r = 0.0; + if (r > 255.0) + r = 255.0; + out[(static_cast(oy) * out_w + ox) * 3 + c] = static_cast(r / 255.0); + } + } + } + return out; +} + +void normalize_inplace(std::vector& x, const JinaVLMProcParams& p) { + if (p.normalization_method == "gaussian") { + for (size_t i = 0; i < x.size(); i += 3) { + for (int c = 0; c < 3; ++c) { + x[i + c] = (x[i + c] - p.image_mean[c]) / p.image_std[c]; + } + } + } else { // minmax: image_min + x * (image_max - image_min) + const float span = p.image_max - p.image_min; + for (float& v : x) { + v = p.image_min + v * span; + } + } +} + +// Patchify an HWC float image [h, w, 3] into [n_patches, patch*patch*3]. +// Matches numpy reshape/transpose in image_processing_jvlm.patchify. +void patchify(const std::vector& src, + int h, + int w, + int patch, + std::vector& dst, + int& n_patches) { + const int hp = h / patch; + const int wp = w / patch; + n_patches = hp * wp; + const int ppp = patch * patch * 3; + dst.assign(static_cast(n_patches) * ppp, 0.0f); + for (int ph = 0; ph < hp; ++ph) { + for (int pw = 0; pw < wp; ++pw) { + const int patch_idx = ph * wp + pw; + float* out = dst.data() + static_cast(patch_idx) * ppp; + int k = 0; + for (int iy = 0; iy < patch; ++iy) { + for (int ix = 0; ix < patch; ++ix) { + const int y = ph * patch + iy; + const int x = pw * patch + ix; + const float* pix = src.data() + (static_cast(y) * w + x) * 3; + out[k++] = pix[0]; + out[k++] = pix[1]; + out[k++] = pix[2]; + } + } + } + } +} + +// Per-patch mask mean for an HW mask [h, w] -> [n_patches]. +void patchify_mask_mean(const std::vector& mask, + int h, + int w, + int patch, + std::vector& dst) { + const int hp = h / patch; + const int wp = w / patch; + dst.assign(static_cast(hp) * wp, 0.0f); + for (int ph = 0; ph < hp; ++ph) { + for (int pw = 0; pw < wp; ++pw) { + double acc = 0.0; + for (int iy = 0; iy < patch; ++iy) { + for (int ix = 0; ix < patch; ++ix) { + acc += mask[static_cast(ph * patch + iy) * w + (pw * patch + ix)]; + } + } + dst[static_cast(ph) * wp + pw] = static_cast(acc / (patch * patch)); + } + } +} + +// Molmo tiling selection (image_processing_jvlm._molmo_select_tiling). +std::pair select_tiling(int h, int w, int crop_window_size, int max_crops) { + std::vector> tilings; + for (int i = 1; i <= max_crops; ++i) { + for (int j = 1; j <= max_crops; ++j) { + if (i * j <= max_crops) { + tilings.emplace_back(i, j); + } + } + } + std::sort(tilings.begin(), tilings.end(), [](const auto& a, const auto& b) { + int pa = a.first * a.second; + int pb = b.first * b.second; + if (pa != pb) + return pa < pb; + return a.first < b.first; + }); + const double orig_h = static_cast(h); + const double orig_w = static_cast(w); + // required_scale per tiling = min over dims of (tiling*cws / original). + std::vector required_scale(tilings.size()); + for (size_t k = 0; k < tilings.size(); ++k) { + double res_h = static_cast(tilings[k].first) * crop_window_size; + double res_w = static_cast(tilings[k].second) * crop_window_size; + double sh = (orig_h != 0.0) ? res_h / orig_h : std::numeric_limits::infinity(); + double sw = (orig_w != 0.0) ? res_w / orig_w : std::numeric_limits::infinity(); + required_scale[k] = std::min(sh, sw); + } + bool all_lt_1 = std::all_of(required_scale.begin(), required_scale.end(), [](double v) { + return v < 1.0; + }); + size_t ix = 0; + if (all_lt_1) { + // argmax + double best = -std::numeric_limits::infinity(); + for (size_t k = 0; k < required_scale.size(); ++k) { + if (required_scale[k] > best) { + best = required_scale[k]; + ix = k; + } + } + } else { + double best = std::numeric_limits::infinity(); + for (size_t k = 0; k < required_scale.size(); ++k) { + double v = required_scale[k] < 1.0 ? 10e9 : required_scale[k]; + if (v < best) { + best = v; + ix = k; + } + } + } + return tilings[ix]; +} + +int get_patches_from_tiling(int num_tiles, + int pooling_size, + int crop_patches, + int crop_window_patches, + int left_margin, + int right_margin) { + auto ceil_to = [](int v, int p) { + return (v + p - 1) / p * p; + }; + if (num_tiles > 1) { + int left = ceil_to(crop_window_patches + left_margin, pooling_size); + int mid = ceil_to(crop_window_patches, pooling_size); + int right = ceil_to(crop_window_patches + right_margin, pooling_size); + return left + (num_tiles - 2) * mid + right; + } + return ceil_to(crop_patches, pooling_size); +} + +} // namespace + +VisionEncoderJVLM::VisionEncoderJVLM(const std::filesystem::path& model_dir, + const std::string& device, + const ov::AnyMap properties) + : VisionEncoder(model_dir, device, properties) { + load_params(model_dir); +} + +VisionEncoderJVLM::VisionEncoderJVLM(const ModelsMap& models_map, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config) + : VisionEncoder(models_map, config_dir_path, device, device_config) { + load_params(config_dir_path); +} + +void VisionEncoderJVLM::load_params(const std::filesystem::path& config_dir_path) { + JinaVLMProcParams p; // defaults + std::ifstream stream(config_dir_path / "preprocessor_config.json"); + if (stream.is_open()) { + nlohmann::json parsed = nlohmann::json::parse(stream); + using ov::genai::utils::read_json_param; + read_json_param(parsed, "patch_size", p.patch_size); + if (parsed.contains("base_input_size")) { + const auto& bis = parsed.at("base_input_size"); + if (bis.is_array() && !bis.empty()) { + p.base_input_size = bis.at(0).get(); + } else if (bis.is_number()) { + p.base_input_size = bis.get(); + } + } + read_json_param(parsed, "max_crops", p.max_crops); + if (parsed.contains("overlap_margins") && parsed.at("overlap_margins").is_array() && + parsed.at("overlap_margins").size() == 2) { + p.overlap_left = parsed.at("overlap_margins").at(0).get(); + p.overlap_right = parsed.at("overlap_margins").at(1).get(); + } + read_json_param(parsed, "pooling_h", p.pooling_h); + read_json_param(parsed, "pooling_w", p.pooling_w); + read_json_param(parsed, "token_length_h", p.token_length_h); + read_json_param(parsed, "token_length_w", p.token_length_w); + read_json_param(parsed, "tokens_per_image", p.tokens_per_image); + read_json_param(parsed, "use_column_tokens", p.use_column_tokens); + read_json_param(parsed, "image_min", p.image_min); + read_json_param(parsed, "image_max", p.image_max); + read_json_param(parsed, "normalization_method", p.normalization_method); + if (parsed.contains("image_mean") && parsed.at("image_mean").is_array() && + parsed.at("image_mean").size() == 3) { + for (int c = 0; c < 3; ++c) + p.image_mean[c] = parsed.at("image_mean").at(c).get(); + } + if (parsed.contains("image_std") && parsed.at("image_std").is_array() && + parsed.at("image_std").size() == 3) { + for (int c = 0; c < 3; ++c) + p.image_std[c] = parsed.at("image_std").at(c).get(); + } + } + m_params = p; +} + +EncodedImage VisionEncoderJVLM::encode(const ov::Tensor& image, const ov::AnyMap& config_map) { + const JinaVLMProcParams& p = m_params; + clip_image_u8 input_image = tensor_to_clip_image_u8(image); + const int H = input_image.ny; + const int W = input_image.nx; + + const int patch = static_cast(p.patch_size); + const int base = static_cast(p.base_input_size); + const int left_margin = static_cast(p.overlap_left); + const int right_margin = static_cast(p.overlap_right); + const int pooling_h = static_cast(p.pooling_h); + const int pooling_w = static_cast(p.pooling_w); + const int TL_h = static_cast(p.token_length_h); + const int TL_w = static_cast(p.token_length_w); + const int total_margin_pixels = patch * (right_margin + left_margin); + const int crop_patches = base / patch; + const int crop_window_patches = crop_patches - (right_margin + left_margin); + const int crop_window_size = crop_window_patches * patch; + + std::pair tiling = + select_tiling(H - total_margin_pixels, W - total_margin_pixels, crop_window_size, static_cast(p.max_crops)); + const int tiling_rows = tiling.first; + const int tiling_cols = tiling.second; + const int rh = tiling_rows * crop_window_size + total_margin_pixels; + const int rw = tiling_cols * crop_window_size + total_margin_pixels; + + // Resize source and normalize. + std::vector src = bilinear_resize_u8_to_unit(input_image, rh, rw); + normalize_inplace(src, p); + // Mask is all-ones (preserve_aspect_ratio == false, no padding). + std::vector src_mask(static_cast(rh) * rw, 1.0f); + + const int image_base_patch = base / patch; // both dims + const int crop_size = base; + const int ppp = patch * patch * 3; + const int n_patches_per_crop = image_base_patch * image_base_patch; + + const int n_tiled_crops = tiling_rows * tiling_cols; + const int n_crops = n_tiled_crops + 1; // + global thumbnail + + // patches[0] is the global thumbnail; patches[1..] are the tiled crops. + ov::Tensor image_patches(ov::element::f32, + ov::Shape{1, static_cast(n_crops), static_cast(n_patches_per_crop), static_cast(ppp)}); + ov::Tensor image_masks(ov::element::i64, + ov::Shape{1, static_cast(n_crops), static_cast(n_patches_per_crop)}); + float* patches_data = image_patches.data(); + int64_t* masks_data = image_masks.data(); + + // Global thumbnail crop (crop index 0 in the tensor layout expected by the IR). + { + std::vector resized = bilinear_resize_u8_to_unit(input_image, base, base); + normalize_inplace(resized, p); + std::vector gpatch; + int np = 0; + patchify(resized, base, base, patch, gpatch, np); + std::copy(gpatch.begin(), gpatch.end(), patches_data); + // image_processing_jvlm builds the mask array from the tiled crops only and then + // pads a single -1 row at the END (np.pad(img_mask, [[0,1],[0,0]], -1)), while the + // thumbnail patches are prepended at the FRONT of the patch array. Reproduce that + // asymmetry: thumbnail mask row goes to the LAST crop slot, tiled masks to slots + // 0..n_tiled_crops-1. The vision IR is exported to consume this exact layout. + int64_t* thumb_mask = masks_data + static_cast(n_crops - 1) * n_patches_per_crop; + for (int i = 0; i < n_patches_per_crop; ++i) { + thumb_mask[i] = -1; + } + } + + // Tiled crops. + // Also compute patch_ordering exactly as image_processing_jvlm.molmo_overlap_and_resize_cropping: + // it numbers pooled patches crop-by-crop into a per-crop token grid (padded with -1 to the + // token_length grid), then transposes to left-to-right order across the whole tiled region. + const int tl_h = TL_h; + const int tl_w = TL_w; + // po_grid holds, for each (tile_row, tile_col, ty, tx), the crop-major running index or -1. + std::vector po_grid(static_cast(tiling_rows) * tiling_cols * tl_h * tl_w, -1); + int on = 0; + int crop_out = 1; + for (int i = 0; i < tiling_rows; ++i) { + const int y0 = i * crop_window_size; + int crop_y0 = (i == 0) ? 0 : (left_margin / pooling_h); + int crop_h = image_base_patch - (right_margin + left_margin); + if (i == 0) + crop_h += left_margin; + if (i == tiling_rows - 1) + crop_h += right_margin; + for (int j = 0; j < tiling_cols; ++j) { + const int x0 = j * crop_window_size; + int crop_x0 = (j == 0) ? 0 : (left_margin / pooling_w); + int crop_w = image_base_patch - (right_margin + left_margin); + if (j == 0) + crop_w += left_margin; + if (j == tiling_cols - 1) + crop_w += right_margin; + int pooled_w = (crop_w + pooling_w - 1) / pooling_w; + int pooled_h = (crop_h + pooling_h - 1) / pooling_h; + // Fill po_grid[i, j, crop_y0:crop_y0+pooled_h, crop_x0:crop_x0+pooled_w] with arange(on, ...) + int running = on; + for (int py = 0; py < pooled_h; ++py) { + for (int px = 0; px < pooled_w; ++px) { + const int ty = crop_y0 + py; + const int tx = crop_x0 + px; + const size_t idx = + (((static_cast(i) * tiling_cols + j) * tl_h) + ty) * tl_w + tx; + po_grid[idx] = running++; + } + } + on += pooled_h * pooled_w; + + // Extract crop [crop_size x crop_size] from src / src_mask. + std::vector crop_img(static_cast(crop_size) * crop_size * 3); + std::vector crop_mask(static_cast(crop_size) * crop_size); + for (int cy = 0; cy < crop_size; ++cy) { + for (int cx = 0; cx < crop_size; ++cx) { + const int sy = y0 + cy; + const int sx = x0 + cx; + for (int c = 0; c < 3; ++c) { + crop_img[(static_cast(cy) * crop_size + cx) * 3 + c] = + src[(static_cast(sy) * rw + sx) * 3 + c]; + } + crop_mask[static_cast(cy) * crop_size + cx] = + src_mask[static_cast(sy) * rw + sx]; + } + } + std::vector cpatch; + int np = 0; + patchify(crop_img, crop_size, crop_size, patch, cpatch, np); + std::copy(cpatch.begin(), + cpatch.end(), + patches_data + static_cast(crop_out) * n_patches_per_crop * ppp); + std::vector mmean; + patchify_mask_mean(crop_mask, crop_size, crop_size, patch, mmean); + // Tiled crop masks occupy slots 0..n_tiled_crops-1 (crop_out is the patch slot, + // which is offset by +1 for the leading thumbnail patch; the mask array is not). + int64_t* crop_mask_out = masks_data + static_cast(crop_out - 1) * n_patches_per_crop; + for (int k = 0; k < n_patches_per_crop; ++k) { + crop_mask_out[k] = static_cast(std::llround(mmean[k])); + } + ++crop_out; + } + } + + // Reproduce the numpy reshape/transpose that reorders patch_ordering into the flat + // crop-major layout used by the vision-output rows, so that valid entries, taken in + // flat order, give left-to-right token order. + // patch_ordering (flat, crop-major) has n_tiled_crops * tl_h * tl_w entries. + const size_t tiled_slots = static_cast(tiling_rows) * tiling_cols * tl_h * tl_w; + std::vector patch_ordering(po_grid); // flat crop-major, values are running indices or -1 + { + // valid positions in the crop-major flat order + std::vector valid_values; + valid_values.reserve(tiled_slots); + for (size_t s = 0; s < tiled_slots; ++s) { + if (patch_ordering[s] >= 0) + valid_values.push_back(patch_ordering[s]); + } + // Build the transposed (left-to-right) order: reshape [tr, tc, tl_h, tl_w] -> transpose to + // [tr, tl_h, tc, tl_w] -> flatten, collect valid values in that order. + std::vector porh_valid; + porh_valid.reserve(valid_values.size()); + for (int i = 0; i < tiling_rows; ++i) { + for (int ty = 0; ty < tl_h; ++ty) { + for (int j = 0; j < tiling_cols; ++j) { + for (int tx = 0; tx < tl_w; ++tx) { + const size_t idx = + (((static_cast(i) * tiling_cols + j) * tl_h) + ty) * tl_w + tx; + if (po_grid[idx] >= 0) + porh_valid.push_back(po_grid[idx]); + } + } + } + } + // patch_ordering[valid] = porh_valid (project transposed order into sparse structure) + size_t vp = 0; + for (size_t s = 0; s < tiled_slots; ++s) { + if (patch_ordering[s] >= 0) { + patch_ordering[s] = porh_valid[vp++]; + } + } + } + + // Prepend the global thumbnail: thumbnail patches map to rows 0..(tokens_per_image-1); + // tiled rows are offset by tokens_per_image. + const int tokens_per_image = static_cast(p.tokens_per_image); + // image_input_idx_flat has length n_crops * tokens_per_image, matching the vision output rows. + // Entry v (>=0) means: this vision row is the v-th slot in joint order. + // Build joint-slot -> vision-row (inverse) then invert to vision-row -> joint-slot rank. + // Thumbnail occupies the first tokens_per_image slots (0..tpi-1) in vision-row order. + // For the tiled region, po_grid running index == tiled vision-row index; joint slot order is the + // flat crop-major slot order (with -1 removed) after the transpose remap. + // We produce image_input_idx per vision row = joint slot rank, or -1 for dropped rows. + ov::Tensor image_input_idx(ov::element::i64, ov::Shape{static_cast(n_crops) * tokens_per_image}); + int64_t* iii_data = image_input_idx.data(); + std::fill(iii_data, iii_data + image_input_idx.get_size(), -1); + // Thumbnail: vision rows 0..tpi-1 -> joint slots 0..tpi-1. + for (int r = 0; r < tokens_per_image; ++r) { + iii_data[r] = r; + } + // Tiled region: iterate joint slots in crop-major flat order; each valid slot's value is the + // tiled vision-row index; the joint-slot rank (its order among valid slots) + tokens_per_image + // is the token position rank. We store, for each tiled vision row, its slot rank. + { + int slot_rank = 0; + for (size_t s = 0; s < tiled_slots; ++s) { + if (patch_ordering[s] >= 0) { + const int64_t tiled_row = patch_ordering[s]; // tiled vision-row index + const size_t vision_row = static_cast(tokens_per_image) + tiled_row; + iii_data[vision_row] = tokens_per_image + slot_rank; + ++slot_rank; + } + } + } + + + // Run the vision embeddings model. + CircularBufferQueueElementGuard infer_request_guard(this->m_ireq_queue_vision_encoder.get()); + ov::InferRequest& encoder = infer_request_guard.get(); + encoder.set_tensor("image_patches", image_patches); + encoder.set_tensor("image_masks", image_masks); + encoder.infer(); + const ov::Tensor& infer_output = encoder.get_output_tensor(); + ov::Tensor image_features(infer_output.get_element_type(), infer_output.get_shape()); + std::memcpy(image_features.data(), infer_output.data(), infer_output.get_byte_size()); + + // Token layout for the tiled crops region (image_processing_jvlm output tokens). + const int h_tok = + get_patches_from_tiling(tiling_rows, pooling_h, crop_patches, crop_window_patches, left_margin, right_margin); + const int w_tok = + get_patches_from_tiling(tiling_cols, pooling_w, crop_patches, crop_window_patches, left_margin, right_margin); + + EncodedImage encoded; + encoded.resized_source = std::move(image_features); // [1, n_crops*196, hidden] + encoded.resized_source_size = {static_cast(h_tok / pooling_h), static_cast(w_tok / pooling_w)}; + encoded.patches_grid = {tiling_rows, tiling_cols}; + encoded.original_image_size = {static_cast(TL_h), static_cast(TL_w)}; + encoded.num_image_tokens = static_cast(infer_output.get_shape().at(1)); + // Carry image_input_idx (per vision-row -> joint slot rank, or -1) for the scatter merge. + encoded.images_features_projection = std::move(image_input_idx); + return encoded; +} + +InputsEmbedderJVLM::InputsEmbedderJVLM(const VLMConfig& vlm_config, + const std::filesystem::path& model_dir, + const Tokenizer& tokenizer, + const std::string& device, + const ov::AnyMap device_config) + : IInputsEmbedder(vlm_config, model_dir, tokenizer, device, device_config) {} + +InputsEmbedderJVLM::InputsEmbedderJVLM(const VLMConfig& vlm_config, + const ModelsMap& models_map, + const Tokenizer& tokenizer, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config) + : IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) {} + +namespace { + +// Build the joint image-token string for a single crop grid (rows x cols), matching +// image_processing_jvlm: + rows * (cols * + ) + . +std::string build_crop_tokens(size_t rows, + size_t cols, + bool use_column_tokens, + const std::string& im_start, + const std::string& im_patch, + const std::string& im_col, + const std::string& im_end) { + std::string s = im_start; + for (size_t r = 0; r < rows; ++r) { + for (size_t c = 0; c < cols; ++c) { + s += im_patch; + } + if (use_column_tokens) { + s += im_col; + } + } + s += im_end; + return s; +} + +} // namespace + +NormalizedPrompt InputsEmbedderJVLM::normalize_prompt(const std::string& prompt, + size_t base_id, + const std::vector& images) const { + const std::string image_prompt_token = m_vlm_config.jvlm_image_prompt_token; // "<|image|>" + const std::string im_start = m_vlm_config.jvlm_image_start_token; // "" + const std::string im_end = m_vlm_config.jvlm_image_end_token; // "" + const std::string im_patch = m_vlm_config.jvlm_image_patch_token; // "" + const std::string im_col = m_vlm_config.jvlm_image_column_token; // "" + + auto [unified_prompt, images_sequence] = normalize(prompt, image_prompt_token, image_prompt_token, base_id, images.size()); + + size_t searched_pos = 0; + for (size_t new_image_id : images_sequence) { + const EncodedImage& enc = images.at(new_image_id - base_id); + const size_t tl_h = enc.original_image_size.height; + const size_t tl_w = enc.original_image_size.width; + const size_t crop_rows = enc.resized_source_size.height; + const size_t crop_cols = enc.resized_source_size.width; + + // Global thumbnail first (token_length_h x token_length_w), then the tiled crops. + std::string expanded = + build_crop_tokens(tl_h, tl_w, /*use_column_tokens=*/true, im_start, im_patch, im_col, im_end); + expanded += build_crop_tokens(crop_rows, crop_cols, /*use_column_tokens=*/true, im_start, im_patch, im_col, im_end); + + searched_pos = unified_prompt.find(image_prompt_token, searched_pos); + OPENVINO_ASSERT(searched_pos != std::string::npos, + "JinaVLM: image placeholder token not found in prompt during normalization"); + unified_prompt.replace(searched_pos, image_prompt_token.length(), expanded); + searched_pos += expanded.length(); + } + return {std::move(unified_prompt), std::move(images_sequence), {}}; +} + +ov::Tensor InputsEmbedderJVLM::build_jvlm_input_ids(const std::string& unified_prompt, + ov::genai::VLMPerfMetrics& metrics) { + // In chat conversation mode the pipeline maintains templated history itself, so + // fall back to the generic path (which encodes the already-templated prompt). + if (m_is_chat_conversation) { + return get_encoded_input_ids(unified_prompt, metrics); + } + + auto encode_start = std::chrono::steady_clock::now(); + + // Reproduce the JinaVLM chat template (chat_template.jinja): + // {{ ' ' }}{{ role.capitalize() + ': ' }}{{ content_text + ' ' }} ... {{ 'Assistant:' }} + // The leading space and the space before "Assistant:" are significant and are + // dropped by the exported OV tokenizer's apply_chat_template, so build it here. + std::string templated_prompt; + bool apply_template = m_apply_chat_template; + if (apply_template) { + templated_prompt = " User: " + unified_prompt + " Assistant:"; + } else { + templated_prompt = unified_prompt; + } + auto template_end_time = std::chrono::steady_clock::now(); + + // The OV tokenizer does not add the BOS token via add_special_tokens; add it + // explicitly to match the reference tokenization used by optimum-intel. + ov::Tensor encoded = + m_tokenizer.encode(templated_prompt, ov::genai::add_special_tokens(false)).input_ids; + auto encode_end = std::chrono::steady_clock::now(); + + if (apply_template) { + metrics.raw_metrics.chat_template_durations.emplace_back( + PerfMetrics::get_microsec(template_end_time - encode_start)); + metrics.raw_metrics.tokenization_durations.emplace_back( + PerfMetrics::get_microsec(encode_end - template_end_time)); + } else { + metrics.raw_metrics.tokenization_durations.emplace_back( + PerfMetrics::get_microsec(encode_end - encode_start)); + } + + // Prepend the BOS token (JinaVLM: <|endoftext|>, config bos_token_id). + const int64_t bos_id = m_tokenizer.get_bos_token_id(); + ov::Tensor new_chat_tokens; + if (bos_id >= 0) { + const auto enc_shape = encoded.get_shape(); + OPENVINO_ASSERT(enc_shape.size() == 2 && enc_shape[0] == 1, + "JinaVLM: unexpected encoded input_ids shape"); + const size_t n = enc_shape[1]; + new_chat_tokens = ov::Tensor(encoded.get_element_type(), ov::Shape{1, n + 1}); + int64_t* dst = new_chat_tokens.data(); + const int64_t* src = encoded.data(); + dst[0] = bos_id; + std::copy_n(src, n, dst + 1); + } else { + new_chat_tokens = encoded; + } + + // Mirror get_encoded_input_ids() history/cache bookkeeping. + ov::Tensor new_input_ids = update_history(new_chat_tokens); + m_prev_hist_length = m_cache_state.get_state().size(); + m_cache_state.add_inputs(new_input_ids); + return new_input_ids; +} + +ov::Tensor InputsEmbedderJVLM::get_inputs_embeds(const std::string& unified_prompt, + const std::vector& images, + ov::genai::VLMPerfMetrics& metrics, + bool recalculate_merged_embeddings, + const std::vector& images_sequence) { + ov::Tensor input_ids = build_jvlm_input_ids(unified_prompt, metrics); + CircularBufferQueueElementGuard embeddings_request_guard(m_embedding->get_request_queue().get()); + EmbeddingsRequest& req = embeddings_request_guard.get(); + ov::Tensor text_embeds = m_embedding->infer(req, input_ids); + + if (images.empty()) { + ov::Tensor inputs_embeds(text_embeds.get_element_type(), text_embeds.get_shape()); + std::memcpy(inputs_embeds.data(), text_embeds.data(), text_embeds.get_byte_size()); + return inputs_embeds; + } + + // Determine the token id. + auto start_tok = std::chrono::steady_clock::now(); + ov::Tensor patch_tok = m_tokenizer.encode(m_vlm_config.jvlm_image_patch_token, ov::genai::add_special_tokens(false)).input_ids; + auto end_tok = std::chrono::steady_clock::now(); + OPENVINO_ASSERT(metrics.raw_metrics.tokenization_durations.size() > 0); + metrics.raw_metrics.tokenization_durations[metrics.raw_metrics.tokenization_durations.size() - 1] += + ov::genai::MicroSeconds(PerfMetrics::get_microsec(end_tok - start_tok)); + const int64_t image_patch_token_id = patch_tok.data()[patch_tok.get_size() - 1]; + + const auto text_shape = text_embeds.get_shape(); + const size_t seq_len = text_shape[1]; + const size_t hidden = text_shape[2]; + const int64_t* ids = input_ids.data(); + + ov::Tensor inputs_embeds(text_embeds.get_element_type(), text_embeds.get_shape()); + std::memcpy(inputs_embeds.data(), text_embeds.data(), text_embeds.get_byte_size()); + float* out = inputs_embeds.data(); + + // For each image (in prompt order), scatter its vision embedding rows onto the + // token positions. image_input_idx[row] gives the joint-slot rank of that vision row (or -1 + // if the row is dropped due to overlap-margin padding). Sorting valid rows by their slot rank + // and placing them at the ordered positions reproduces the reference + // image_input_idx scatter exactly (validated in .model_enabler/jvlm/proto_multicrop_merge.py). + size_t token_cursor = 0; + for (size_t new_image_id : images_sequence) { + const EncodedImage& enc = images.at(new_image_id); + const ov::Tensor& feats = enc.resized_source; // [1, n_rows, hidden] + const size_t n_rows = feats.get_shape().at(1); + const float* feat_data = feats.data(); + const ov::Tensor& iii = enc.images_features_projection; + OPENVINO_ASSERT(iii && iii.get_size() == n_rows, + "JinaVLM: image_input_idx size does not match vision embedding rows"); + const int64_t* iii_data = iii.data(); + + // Collect valid (slot_rank, vision_row) pairs and sort by slot_rank. + std::vector> valid; + valid.reserve(n_rows); + for (size_t r = 0; r < n_rows; ++r) { + if (iii_data[r] >= 0) { + valid.emplace_back(iii_data[r], r); + } + } + std::sort(valid.begin(), valid.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + + // Place each valid vision row at the next position. + size_t placed = 0; + for (; token_cursor < seq_len && placed < valid.size(); ++token_cursor) { + if (ids[token_cursor] == image_patch_token_id) { + const size_t vision_row = valid[placed].second; + std::copy_n(feat_data + vision_row * hidden, hidden, out + token_cursor * hidden); + ++placed; + } + } + OPENVINO_ASSERT(placed == valid.size(), + "JinaVLM: number of tokens does not match valid vision embedding rows"); + } + return inputs_embeds; +} + +} // namespace ov::genai diff --git a/src/cpp/src/visual_language/jvlm/classes.hpp b/src/cpp/src/visual_language/jvlm/classes.hpp new file mode 100644 index 0000000000..3116259303 --- /dev/null +++ b/src/cpp/src/visual_language/jvlm/classes.hpp @@ -0,0 +1,115 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "visual_language/vlm_config.hpp" +#include "visual_language/vision_encoder.hpp" +#include "visual_language/inputs_embedder.hpp" + +namespace ov::genai { + +// Preprocessing parameters for the JinaVLM (jvlm) Molmo-style image processor. +// Loaded from preprocessor_config.json. +struct JinaVLMProcParams { + size_t patch_size = 14; + size_t base_input_size = 378; // both dims equal in known configs + size_t max_crops = 12; + size_t overlap_left = 4; + size_t overlap_right = 4; + size_t pooling_h = 2; + size_t pooling_w = 2; + size_t token_length_h = 14; + size_t token_length_w = 14; + size_t tokens_per_image = 196; + bool use_column_tokens = true; + std::array image_mean = {0.48145466f, 0.4578275f, 0.40821073f}; + std::array image_std = {0.26862954f, 0.26130258f, 0.27577711f}; + float image_min = -1.0f; + float image_max = 1.0f; + std::string normalization_method = "minmax"; // or "gaussian" +}; + +// Result of JinaVLM image preprocessing for a single image. +struct JinaVLMPreprocessed { + // [n_crops, n_patches, patch*patch*3] + ov::Tensor image_patches; + // [n_crops, n_patches] (float mean of per-pixel mask, values in {-1, 1}) + ov::Tensor image_masks; + // Number of crops (including the leading global thumbnail crop). + size_t n_crops = 0; + // Per-crop tiling of the non-thumbnail crops (rows, cols) used for prompt token layout. + size_t tiling_rows = 1; + size_t tiling_cols = 1; + // Number of token rows/cols emitted for the tiled crops region. + size_t crop_rows = 0; // h // pooling_h + size_t crop_cols = 0; // w // pooling_w +}; + +class VisionEncoderJVLM : public VisionEncoder { +public: + VisionEncoderJVLM(const std::filesystem::path& model_dir, + const std::string& device, + const ov::AnyMap properties); + + VisionEncoderJVLM(const ModelsMap& models_map, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config); + + EncodedImage encode(const ov::Tensor& image, const ov::AnyMap& config_map) override; + + const JinaVLMProcParams& get_jvlm_params() const { return m_params; } + +private: + void load_params(const std::filesystem::path& config_dir_path); + JinaVLMProcParams m_params; +}; + +class InputsEmbedderJVLM : public InputsEmbedder::IInputsEmbedder { +public: + InputsEmbedderJVLM( + const VLMConfig& vlm_config, + const std::filesystem::path& model_dir, + const Tokenizer& tokenizer, + const std::string& device, + const ov::AnyMap device_config); + + InputsEmbedderJVLM( + const VLMConfig& vlm_config, + const ModelsMap& models_map, + const Tokenizer& tokenizer, + const std::filesystem::path& config_dir_path, + const std::string& device, + const ov::AnyMap device_config); + + ov::Tensor get_inputs_embeds(const std::string& prompt, + const std::vector& images, + ov::genai::VLMPerfMetrics& metrics, + bool recalculate_merged_embeddings = true, + const std::vector& image_sequence = {}) override; + + NormalizedPrompt normalize_prompt( + const std::string& prompt, + size_t base_id, + const std::vector& images) const override; + +private: + // JinaVLM-specific input-id construction. + // + // The JinaVLM chat template renders " User: Assistant:" with a + // leading space and relies on the tokenizer to prepend the BOS token + // (<|endoftext|>, id from Tokenizer::get_bos_token_id). The exported + // OpenVINO tokenizer neither preserves that leading space through + // apply_chat_template nor adds the BOS token via add_special_tokens, so the + // generic get_encoded_input_ids() path diverges from the reference + // HF/optimum-intel tokenization. This helper reproduces the reference + // tokenization exactly: it wraps the (already image-expanded) prompt in the + // JinaVLM template when chat templating is requested, encodes with + // add_special_tokens(false), and prepends the BOS token id. + ov::Tensor build_jvlm_input_ids(const std::string& unified_prompt, ov::genai::VLMPerfMetrics& metrics); +}; + +} // namespace ov::genai diff --git a/src/cpp/src/visual_language/vision_encoder.cpp b/src/cpp/src/visual_language/vision_encoder.cpp index 40daf8d967..58799527ee 100644 --- a/src/cpp/src/visual_language/vision_encoder.cpp +++ b/src/cpp/src/visual_language/vision_encoder.cpp @@ -25,9 +25,9 @@ #include "visual_language/gemma3n/classes.hpp" #include "visual_language/gemma4/classes.hpp" #include "visual_language/videochat_flash/classes.hpp" +#include "visual_language/jvlm/classes.hpp" namespace ov::genai { - VisionEncoder::VisionEncoder(const std::filesystem::path& model_dir, const std::string& device, const ov::AnyMap properties) { auto compiled_model = utils::singleton_core().compile_model( model_dir / "openvino_vision_embeddings_model.xml", device, @@ -146,6 +146,8 @@ VisionEncoder::Ptr VisionEncoder::create(const std::filesystem::path& model_dir, return std::make_shared(model_dir, device, properties); } else if (model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { return std::make_shared(model_dir, device, properties); + } else if (model_type == VLMModelType::JVLM) { + return std::make_shared(model_dir, device, properties); } else { OPENVINO_THROW("Unsupported model type in VLM VisionEncoder class. Please, create feature request on new model support"); } @@ -193,6 +195,8 @@ VisionEncoder::Ptr VisionEncoder::create( return std::make_shared(models_map, config_dir_path, device, device_config); } else if (model_type == VLMModelType::VIDEOCHAT_FLASH_QWEN) { return std::make_shared(models_map, config_dir_path, device, device_config); + } else if (model_type == VLMModelType::JVLM) { + return std::make_shared(models_map, config_dir_path, device, device_config); } else { OPENVINO_THROW("Unsupported model type in VLM VisionEncoder class. Please, create feature request on new model support"); } diff --git a/src/cpp/src/visual_language/vlm_config.cpp b/src/cpp/src/visual_language/vlm_config.cpp index 0846e52bfa..1c490051d4 100644 --- a/src/cpp/src/visual_language/vlm_config.cpp +++ b/src/cpp/src/visual_language/vlm_config.cpp @@ -34,6 +34,7 @@ VLMModelType to_vlm_model_type(const std::string& value) { {"videochat_flash_qwen", VLMModelType::VIDEOCHAT_FLASH_QWEN}, {"qwen3_omni", VLMModelType::QWEN3_OMNI}, {"qwen3_omni_moe", VLMModelType::QWEN3_OMNI}, + {"jvlm", VLMModelType::JVLM}, }; auto it = model_types_map.find(value); @@ -62,6 +63,10 @@ VLMConfig::VLMConfig(const std::filesystem::path& json_path) { read_json_param(parsed, "query_num", query_num); read_json_param(parsed, "use_image_id", use_image_id); + if (model_type == VLMModelType::JVLM) { + read_json_param(parsed, "text_config.hidden_size", hidden_size); + } + read_json_param(parsed, "image_newline", image_newline); read_json_param(parsed, "vision_config.patch_size", vision_config_patch_size); diff --git a/src/cpp/src/visual_language/vlm_config.hpp b/src/cpp/src/visual_language/vlm_config.hpp index 6552e77199..7fb2e1355d 100644 --- a/src/cpp/src/visual_language/vlm_config.hpp +++ b/src/cpp/src/visual_language/vlm_config.hpp @@ -32,6 +32,7 @@ enum class VLMModelType { GEMMA4_UNIFIED, VIDEOCHAT_FLASH_QWEN, QWEN3_OMNI, + JVLM, }; /// @brief A Configuration class passed to VLMPipeline and used to @@ -163,6 +164,18 @@ class VLMConfig { // Speaker name-to-codec-token mapping std::map speaker_ids; + /// @brief JinaVLM (jvlm) image special tokens. + /// Placeholder for a whole image in the prompt (replaced during normalization). + std::string jvlm_image_prompt_token = "<|image|>"; + /// @brief Per-patch placeholder token where vision embeddings are inserted. + std::string jvlm_image_patch_token = ""; + /// @brief Column separator token appended after each row of patches. + std::string jvlm_image_column_token = ""; + /// @brief Image region start token. + std::string jvlm_image_start_token = ""; + /// @brief Image region end token. + std::string jvlm_image_end_token = ""; + /// @brief Default constructor. VLMConfig() = default; /// @brief Construct VLMConfig from values in json_path. diff --git a/tests/python_tests/test_vlm_pipeline.py b/tests/python_tests/test_vlm_pipeline.py index 9ebee38bac..8761a4c08f 100644 --- a/tests/python_tests/test_vlm_pipeline.py +++ b/tests/python_tests/test_vlm_pipeline.py @@ -167,6 +167,7 @@ def __getattr__(self, name: str): MODEL_GEMMA = "optimum-intel-internal-testing/tiny-random-gemma3" MODEL_GEMMA3N = "optimum-intel-internal-testing/tiny-random-gemma3n" MODEL_QWEN3_OMNI = "optimum-intel-internal-testing/tiny-random-qwen3-omni" +MODEL_JVLM = "optimum-intel-internal-testing/tiny-random-jvlm" MODEL_IDS: list[str] = [] if is_transformers_version("<", "5.0"): @@ -181,6 +182,9 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-gemma3", MODEL_GEMMA3N, "optimum-intel-internal-testing/tiny-random-MiniCPM-o-2_6", + # JinaVLM (model_type='jvlm', Molmo-style overlap-and-resize preprocessing). + # Requires transformers>=4.57 and trust_remote_code (remote modeling/processing code). + MODEL_JVLM, *VIDEO_MODEL_IDS, ] else: @@ -217,6 +221,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-gemma4-unified-it": lambda idx: "<|image|>", "optimum-intel-internal-testing/tiny-random-gemma4-31B": lambda idx: "<|image|>", "qnguyen3/nanoLLaVA": lambda idx: "\n", + MODEL_JVLM: lambda idx: "<|image|>", VIDEOCHAT_FLASH_QWEN_MODEL_ID: lambda idx: f"<|image_{idx + 1}|>\n", } @@ -244,6 +249,7 @@ def __getattr__(self, name: str): "optimum-intel-internal-testing/tiny-random-qwen2.5-vl": 336, "optimum-intel-internal-testing/tiny-random-qwen3-vl": 256, "optimum-intel-internal-testing/tiny-random-qwen3.5": 256, + MODEL_JVLM: 64, } diff --git a/tests/python_tests/utils/hugging_face.py b/tests/python_tests/utils/hugging_face.py index fd188d44c5..bbb2dc1093 100644 --- a/tests/python_tests/utils/hugging_face.py +++ b/tests/python_tests/utils/hugging_face.py @@ -341,7 +341,7 @@ def sanitize_model_id(model_id: str) -> str: return model_id.replace("/", "_") -TRUST_REMOTE_CODE_MODELS = ("AngelSlim/Qwen3-1.7B_eagle3", "optimum-intel-internal-testing/tiny-random-qwen3-vl-eagle3") +TRUST_REMOTE_CODE_MODELS = ("AngelSlim/Qwen3-1.7B_eagle3", "optimum-intel-internal-testing/tiny-random-qwen3-vl-eagle3", "optimum-intel-internal-testing/tiny-random-jvlm") # Some models require optimum-cli export instead of the Python API path. # This maps model_id to the --task value used during export - CVS-183496 diff --git a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py index 453ed6f025..2a50f5b616 100644 --- a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py +++ b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/__init__.py @@ -8,6 +8,7 @@ from .qwen3 import Qwen3VLInputsPreprocessor, Qwen3_5VLInputsPreprocessor, Qwen3OmniInputsPreprocessor from .gemma3 import Gemma3InputsPreprocessor from .gemma4 import Gemma4InputsPreprocessor, Gemma4UnifiedInputsPreprocessor, Gemma3nInputsPreprocessor +from .jvlm import JinaVLMInputsPreprocessor from .vlm_inputs_preprocessor import VLMInputsPreprocessor MODEL_TYPE_TO_CLS_MAPPING = { @@ -33,6 +34,7 @@ "llava_next": LLAVAInputsPreprocessor, "llava-qwen2": NanoLlavaInputsPreprocessor, "internvl_chat": InternVLInputsPreprocessor, + "jvlm": JinaVLMInputsPreprocessor, } __all__ = ["MODEL_TYPE_TO_CLS_MAPPING", "VLMInputsPreprocessor"] diff --git a/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/jvlm.py b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/jvlm.py new file mode 100644 index 0000000000..b5e9637f2b --- /dev/null +++ b/tools/who_what_benchmark/whowhatbench/inputs_preprocessors/jvlm.py @@ -0,0 +1,87 @@ +import numpy as np +from transformers import ( + AutoImageProcessor, + PretrainedConfig, + PreTrainedTokenizer, +) +from .vlm_inputs_preprocessor import VLMInputsPreprocessor +from typing import TYPE_CHECKING, Optional, Union, Any + +if TYPE_CHECKING: + from PIL.Image import Image + from transformers.image_utils import VideoInput + + +class JinaVLMInputsPreprocessor(VLMInputsPreprocessor): + """Inputs preprocessor for the JinaVLM family (model_type='jvlm', + JinaVLMForConditionalGeneration). + + JinaVLM ships a dedicated `JinaVLMProcessor` that wraps an image processor + and a tokenizer. It expects the image placeholder token (`<|image|>`) to be + present in the text; this is produced by the model's chat template for + `content` entries of type `image`. The processor then builds + `input_ids`, `image_patches`, `image_input_idx`, `image_masks` and + `attention_mask`, which the model's `generate()` consumes directly. + """ + + def __init__(self, chat_mode: bool = False, model: Optional[Any] = None): + super().__init__(chat_mode) + # JinaVLM interleaves image features by an image placeholder token id. + # Prefer the processor-derived id when available; fall back to config. + if model is not None: + self.def_image_token_id = getattr(model.config, "image_token_id", None) + else: + self.def_image_token_id = None + + def update_chat_history_with_answer(self, answer): + self.chat_history.append({"role": "assistant", "content": [{"type": "text", "text": answer}]}) + + def preprocess_inputs( + self, + text: str, + image: Optional[Union["Image", list["Image"]]] = None, + processor: Optional[AutoImageProcessor] = None, + tokenizer: Optional[PreTrainedTokenizer] = None, + config: Optional[PretrainedConfig] = None, + video: Optional[Union["VideoInput", list["VideoInput"]]] = None, + audio: Optional[np.ndarray] = None, + ): + if processor is None: + raise ValueError("Processor is required.") + if video is not None: + raise ValueError("Video input is not supported") + if audio is not None: + raise ValueError("Audio input is not supported") + if getattr(processor, "chat_template", None) is None: + raise ValueError("JinaVLM requires a chat template to build image-text inputs.") + + if image is not None and not isinstance(image, list): + image = [image] + + self.update_images(image) + + content = [] + if image is not None: + content.extend([{"type": "image"}] * len(image)) + content.append({"type": "text", "text": text}) + + new_message = {"role": "user", "content": content} + if self.chat_mode: + self.chat_history.append(new_message) + conversation = self.chat_history + else: + conversation = [new_message] + + prompt = processor.apply_chat_template( + conversation, + add_generation_prompt=True, + tokenize=False, + ) + + inputs = processor( + images=self.images, + text=prompt, + return_tensors="pt", + ) + + return inputs diff --git a/tools/who_what_benchmark/whowhatbench/model_loaders.py b/tools/who_what_benchmark/whowhatbench/model_loaders.py index 9896ba7b2f..8482338c0f 100644 --- a/tools/who_what_benchmark/whowhatbench/model_loaders.py +++ b/tools/who_what_benchmark/whowhatbench/model_loaders.py @@ -510,6 +510,12 @@ def load_visual_text_model( elif config.model_type == "gemma3n": model_cls = AutoModelForCausalLM model_kwargs.update({"torch_dtype": torch.float32}) + elif config.model_type == "jvlm": + # JinaVLM (JinaVLMForConditionalGeneration) exposes generation via + # its AutoModelForCausalLM auto_map entry. The AutoModelForImageTextToText/ + # AutoModelForVision2Seq resolution returns the base JinaVLM class, + # which has no `generate`, so select the causal-LM class explicitly. + model_cls = AutoModelForCausalLM elif transformers_version < Version("5.0.0"): from transformers import AutoModelForVision2Seq diff --git a/tools/who_what_benchmark/whowhatbench/wwb.py b/tools/who_what_benchmark/whowhatbench/wwb.py index 9b88837dd3..6b753ab54c 100644 --- a/tools/who_what_benchmark/whowhatbench/wwb.py +++ b/tools/who_what_benchmark/whowhatbench/wwb.py @@ -518,9 +518,68 @@ def check_args(args): raise ValueError("--llamacpp-chat requires --llamacpp") +def _load_local_csv_prompts(csv_path, model_type): + """Load prompts from a local CSV file. + + For visual-text / visual-video-text tasks the CSV may carry the + multimodal columns ``prompts``, ``images`` and ``videos``. Image (and + video) paths are resolved deterministically relative to the CSV location + when they are not absolute, so a self-contained dataset directory works + regardless of the current working directory. Any other task falls back to + the single ``prompts`` column. + """ + import pandas as pd + from transformers.image_utils import load_image + + df = pd.read_csv(csv_path, keep_default_na=False) + if "prompts" not in df.columns: + raise ValueError( + f"Local dataset CSV '{csv_path}' must contain a 'prompts' column, " + f"got columns {list(df.columns)}." + ) + + prompts = [p for p in df["prompts"].tolist()] + + is_visual = model_type in ("visual-text", "visual-video-text", "visual-text-chat") + if not is_visual: + return {"prompts": prompts} + + base_dir = os.path.dirname(os.path.abspath(csv_path)) + + def _resolve(entry): + entry = "" if entry is None else str(entry).strip() + if entry == "": + return None + path = entry if os.path.isabs(entry) else os.path.join(base_dir, entry) + return path + + images = None + if "images" in df.columns: + images = [ + load_image(resolved) if (resolved := _resolve(v)) is not None else None + for v in df["images"].tolist() + ] + + videos = None + if "videos" in df.columns: + videos = [_resolve(v) for v in df["videos"].tolist()] + + res = {"prompts": prompts} + res["images"] = images if images is not None else [None] * len(prompts) + res["videos"] = videos if videos is not None else [None] * len(prompts) + return res + + def load_prompts(args): if args.dataset is None: return None + + # A local CSV file is a valid dataset source. This keeps a single + # generic --dataset interface while allowing offline/deterministic input + # data (e.g. when the default remote dataset is unavailable or too slow). + if os.path.isfile(args.dataset) and args.dataset.lower().endswith(".csv"): + return _load_local_csv_prompts(args.dataset, args.model_type) + split = "validation" if args.split is not None: split = args.split