feat(qwen3): bundle Qwen3 tokenizer for offline single-file/GGUF encoders

invokeai/backend/model_manager/load/model_loaders/z_image.py

@@ -28,6 +28,7 @@
     SubModelType,
 )
 from invokeai.backend.quantization.gguf.loaders import gguf_sd_loader
+from invokeai.backend.qwen3.qwen3_tokenizer import load_bundled_qwen3_tokenizer
 from invokeai.backend.util.devices import TorchDevice
 
 
@@ -533,10 +534,6 @@ def _load_control_adapter(
 class Qwen3EncoderCheckpointLoader(ModelLoader):
     """Class to load single-file Qwen3 Encoder models for Z-Image (safetensors format)."""
 
-    # Default HuggingFace model to load tokenizer from when using single-file Qwen3 encoder
-    # Must be Qwen3 (not Qwen2.5) to match Z-Image's text encoder architecture and special tokens
-    DEFAULT_TOKENIZER_SOURCE = "Qwen/Qwen3-4B"
-
     def _load_model(
         self,
         config: AnyModelConfig,
@@ -558,18 +555,14 @@ def _load_model(
         )
 
     def _load_tokenizer_with_offline_fallback(self) -> AnyModel:
-        """Load tokenizer with local_files_only fallback for offline support.
+        """Load the Qwen3 tokenizer from the vendored, bundled copy.
 
-        First tries to load from local cache (offline), falling back to network download
-        if the tokenizer hasn't been cached yet. This ensures offline operation after
-        the initial download.
+        Single-file / GGUF checkpoints do not ship tokenizer files. The Qwen3 BPE
+        tokenizer is identical across the 0.6B / 4B / 8B variants, so we load the
+        self-contained copy vendored in the package — fully offline, no HuggingFace
+        download required.
         """
-        try:
-            # Try loading from local cache first (supports offline usage)
-            return AutoTokenizer.from_pretrained(self.DEFAULT_TOKENIZER_SOURCE, local_files_only=True)
-        except OSError:
-            # Not in cache yet, download from HuggingFace
-            return AutoTokenizer.from_pretrained(self.DEFAULT_TOKENIZER_SOURCE)
+        return load_bundled_qwen3_tokenizer()
 
     def _load_from_singlefile(
         self,
@@ -790,10 +783,6 @@ def _load_from_singlefile(
 class Qwen3EncoderGGUFLoader(ModelLoader):
     """Class to load GGUF-quantized Qwen3 Encoder models for Z-Image."""
 
-    # Default HuggingFace model to load tokenizer from when using GGUF Qwen3 encoder
-    # Must be Qwen3 (not Qwen2.5) to match Z-Image's text encoder architecture and special tokens
-    DEFAULT_TOKENIZER_SOURCE = "Qwen/Qwen3-4B"
-
     def _load_model(
         self,
         config: AnyModelConfig,
@@ -815,18 +804,14 @@ def _load_model(
         )
 
     def _load_tokenizer_with_offline_fallback(self) -> AnyModel:
-        """Load tokenizer with local_files_only fallback for offline support.
+        """Load the Qwen3 tokenizer from the vendored, bundled copy.
 
-        First tries to load from local cache (offline), falling back to network download
-        if the tokenizer hasn't been cached yet. This ensures offline operation after
-        the initial download.
+        Single-file / GGUF checkpoints do not ship tokenizer files. The Qwen3 BPE
+        tokenizer is identical across the 0.6B / 4B / 8B variants, so we load the
+        self-contained copy vendored in the package — fully offline, no HuggingFace
+        download required.
         """
-        try:
-            # Try loading from local cache first (supports offline usage)
-            return AutoTokenizer.from_pretrained(self.DEFAULT_TOKENIZER_SOURCE, local_files_only=True)
-        except OSError:
-            # Not in cache yet, download from HuggingFace
-            return AutoTokenizer.from_pretrained(self.DEFAULT_TOKENIZER_SOURCE)
+        return load_bundled_qwen3_tokenizer()
 
     def _load_from_gguf(
         self,

invokeai/backend/qwen3/__init__.py

@@ -0,0 +1,6 @@
+"""Qwen3 encoder backend module.
+
+Shared assets for the standalone Qwen3 text encoders used by Z-Image (4B/8B) and
+Anima (0.6B). The Qwen3 BPE tokenizer is identical across all variants, so a single
+vendored copy is bundled here and reused by every Qwen3 encoder loader.
+"""

invokeai/backend/qwen3/qwen3_tokenizer.py

@@ -0,0 +1,22 @@
+"""Bundled Qwen3 tokenizer for single-file / GGUF Qwen3 encoders.
+
+Single-file (safetensors) and GGUF Qwen3 encoder checkpoints ship weights only —
+no tokenizer files. Previously the tokenizer was pulled from HuggingFace
+(``Qwen/Qwen3-4B``) on first use, which fails in offline / airgapped setups and
+whenever the HF cache is not persisted. The Qwen3 BPE tokenizer is identical
+across the 0.6B / 4B / 8B variants, so a single self-contained copy (Apache-2.0,
+vendored from ``Qwen/Qwen3-4B``) serves every Qwen3 encoder, fully offline.
+"""
+
+from functools import lru_cache
+from pathlib import Path
+
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
+
+_TOKENIZER_DIR = Path(__file__).parent / "tokenizer"
+
+
+@lru_cache(maxsize=1)
+def load_bundled_qwen3_tokenizer() -> PreTrainedTokenizerBase:
+    """Load the vendored Qwen3 fast tokenizer. Result is cached for the process."""
+    return AutoTokenizer.from_pretrained(_TOKENIZER_DIR, local_files_only=True)

invokeai/backend/qwen3/tokenizer/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}