diff --git a/.github/workflows/bump-tokenizers.yml b/.github/workflows/bump-tokenizers.yml new file mode 100644 index 00000000..947a2208 --- /dev/null +++ b/.github/workflows/bump-tokenizers.yml @@ -0,0 +1,100 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Keep Hugging Face tokenizers current (docs/tokenizers.md): when +# huggingface/tokenizers publishes a newer release, open a PR that moves +# third_party/tokenizers to it and refreshes hf_tokenizers/Cargo.lock. Before +# merging, regenerate the golden cases with the same Python release +# (tools/make_tokenizer_golden.py) and run the hf_tokenizer tests. +# +# Uses the secret HRX_BUMP_TOKEN (Contents and Pull requests read/write on +# 1bit-MONSTER/engine): a PR opened with the default GITHUB_TOKEN would not +# start CI. +name: bump-tokenizers + +on: + schedule: + - cron: "5 8 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: bump-tokenizers + cancel-in-progress: false + +jobs: + bump: + runs-on: ubuntu-latest + steps: + - name: Require the token + env: + HRX_BUMP_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }} + run: | + if [ -z "$HRX_BUMP_TOKEN" ]; then + echo "::error::secret HRX_BUMP_TOKEN is not set (see the header of this workflow)" + exit 1 + fi + + - uses: actions/checkout@v4 + with: + token: ${{ secrets.HRX_BUMP_TOKEN }} + submodules: false + + - name: Compare with the latest release + id: pins + env: + GH_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }} + run: | + set -euo pipefail + tag=$(gh api repos/huggingface/tokenizers/releases/latest --jq .tag_name) + upstream=$(git ls-remote https://github.com/huggingface/tokenizers.git "refs/tags/${tag}^{}" | awk '{print $1}') + [ -n "$upstream" ] || upstream=$(git ls-remote https://github.com/huggingface/tokenizers.git "refs/tags/${tag}" | awk '{print $1}') + ours=$(git ls-tree HEAD third_party/tokenizers | awk '{print $3}') + echo "latest $tag = $upstream, ours $ours" + { + echo "tag=$tag"; echo "upstream=$upstream"; echo "ours=$ours" + if [ "$upstream" = "$ours" ]; then echo "changed=false"; else echo "changed=true"; fi + } >> "$GITHUB_OUTPUT" + + - name: Open the bump PR + if: steps.pins.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }} + TAG: ${{ steps.pins.outputs.tag }} + UPSTREAM: ${{ steps.pins.outputs.upstream }} + OURS: ${{ steps.pins.outputs.ours }} + run: | + set -euo pipefail + branch="bump-tokenizers/${TAG}" + if git ls-remote --exit-code origin "refs/heads/$branch" > /dev/null; then + echo "$branch already exists"; exit 0 + fi + git config user.name "tokenizers-bump" + git config user.email "tokenizers-bump@users.noreply.github.com" + git switch -c "$branch" + git submodule update --init --depth 1 third_party/tokenizers + git -C third_party/tokenizers fetch -q --depth 1 origin "$UPSTREAM" + git -C third_party/tokenizers checkout -q "$UPSTREAM" + (cd hf_tokenizers && cargo update -p tokenizers) + git add third_party/tokenizers hf_tokenizers/Cargo.lock + git commit -q -m "Bump tokenizers: huggingface/tokenizers ${TAG} (${UPSTREAM:0:12})" + git push -q origin "$branch" + gh pr create --base main --head "$branch" \ + --title "Bump tokenizers: ${TAG}" \ + --body "Moves third_party/tokenizers from \`${OURS:0:12}\` to \`${TAG}\` (\`${UPSTREAM:0:12}\`) and refreshes hf_tokenizers/Cargo.lock. + + Before merging: regenerate tests/data/tokenizers/*.json with Python tokenizers of the same release (tools/make_tokenizer_golden.py) and run ctest -R hf_tokenizer with -DONEBIT_TOKENIZER_ROOT." diff --git a/.gitmodules b/.gitmodules index 8de1a8d2..6e775637 100644 --- a/.gitmodules +++ b/.gitmodules @@ -48,4 +48,10 @@ path = third_party/laya url = https://github.com/NandhaKishorM/laya.git branch = main +# Hugging Face tokenizers (docs/tokenizers.md): the Rust library behind the +# tokenizers npm/PyPI packages, same release; built by hf_tokenizers/ through +# CMake (-DONEBIT_HF_TOKENIZERS=ON); .github/workflows/bump-tokenizers.yml keeps it current. +[submodule "third_party/tokenizers"] + path = third_party/tokenizers + url = https://github.com/huggingface/tokenizers.git shallow = true diff --git a/CMakeLists.txt b/CMakeLists.txt index 045512ec..41e8b4cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,31 @@ if(ONEBIT_ZINC) COMMAND ${CMAKE_SOURCE_DIR}/tests/zinc_lemonade_e2e.sh $ ${ONEBIT_ZINC_SERVER}) endif() +# ── Hugging Face tokenizers (docs/tokenizers.md) ─────────────────────────── +# Off by default: it needs cargo. Any model's tokenizer.json, through our C ABI +# over third_party/tokenizers (the same release as the tokenizers npm/PyPI +# packages). tests/data/tokenizers/ holds golden cases for 18 models. +option(ONEBIT_HF_TOKENIZERS "Build Hugging Face tokenizers (Rust) behind a C ABI" OFF) +if(ONEBIT_HF_TOKENIZERS) + find_package(Threads REQUIRED) + include(cmake/hf_tokenizers.cmake) + add_executable(hf_tokenizer_test tests/hf_tokenizer_test.cpp) + target_link_libraries(hf_tokenizer_test PRIVATE onebit_hf_tokenizers nlohmann_json::nlohmann_json) + # Golden cases whose tokenizer.json is under a model root (ONEBIT_TOKENIZER_ROOT, + # e.g. ~/.config/flm/models: //tokenizer.json). + set(ONEBIT_TOKENIZER_ROOT "" CACHE PATH "Directory of model dirs holding tokenizer.json, for hf_tokenizer tests") + if(ONEBIT_TOKENIZER_ROOT) + file(GLOB _tok_golden "${CMAKE_SOURCE_DIR}/tests/data/tokenizers/*.json") + foreach(_g ${_tok_golden}) + get_filename_component(_m "${_g}" NAME_WLE) + if(EXISTS "${ONEBIT_TOKENIZER_ROOT}/${_m}/tokenizer.json") + add_test(NAME hf_tokenizer_${_m} + COMMAND hf_tokenizer_test ${ONEBIT_TOKENIZER_ROOT}/${_m}/tokenizer.json ${_g}) + endif() + endforeach() + endif() +endif() + # ── Step 3: NPU engine, full ELFs only (docs/npu.md) ────────────────────── # Host side (full-ELF generation, model reader, buffer packing): pure C++, no # XRT. It needs PCRE2 (the tokenizer), and macOS has no NPU, so it is off there diff --git a/cmake/hf_tokenizers.cmake b/cmake/hf_tokenizers.cmake new file mode 100644 index 00000000..baff99b9 --- /dev/null +++ b/cmake/hf_tokenizers.cmake @@ -0,0 +1,40 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Hugging Face tokenizers (docs/tokenizers.md): builds hf_tokenizers/ (our C ABI +# over third_party/tokenizers) with cargo, --locked, into the build tree, and +# exposes it as the onebit_hf_tokenizers target. rustup honours +# hf_tokenizers/rust-toolchain.toml, which pins the Rust release. +find_program(CARGO cargo REQUIRED) +if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/tokenizers/tokenizers/Cargo.toml") + message(FATAL_ERROR "ONEBIT_HF_TOKENIZERS needs third_party/tokenizers: git submodule update --init third_party/tokenizers") +endif() +set(_hf_target_dir "${CMAKE_BINARY_DIR}/hf_tokenizers") +set(_hf_lib "${_hf_target_dir}/release/${CMAKE_STATIC_LIBRARY_PREFIX}onebit_hf_tokenizers${CMAKE_STATIC_LIBRARY_SUFFIX}") +file(GLOB_RECURSE _hf_sources CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/hf_tokenizers/src/*.rs") +add_custom_command( + OUTPUT "${_hf_lib}" + COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_hf_target_dir} + ${CARGO} build --release --locked --manifest-path ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.toml + DEPENDS ${_hf_sources} ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.toml ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.lock + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/hf_tokenizers + COMMENT "cargo build hf_tokenizers (Hugging Face tokenizers)" + USES_TERMINAL) +add_custom_target(hf_tokenizers_build DEPENDS "${_hf_lib}") +add_library(onebit_hf_tokenizers STATIC IMPORTED GLOBAL) +set_target_properties(onebit_hf_tokenizers PROPERTIES IMPORTED_LOCATION "${_hf_lib}") +target_include_directories(onebit_hf_tokenizers INTERFACE "${CMAKE_SOURCE_DIR}/hf_tokenizers/include") +target_link_libraries(onebit_hf_tokenizers INTERFACE Threads::Threads ${CMAKE_DL_LIBS} m) +add_dependencies(onebit_hf_tokenizers hf_tokenizers_build) diff --git a/docs/tokenizers.md b/docs/tokenizers.md new file mode 100644 index 00000000..35290f0d --- /dev/null +++ b/docs/tokenizers.md @@ -0,0 +1,56 @@ + +# Hugging Face tokenizers + +The engine reads any model's `tokenizer.json` through Hugging Face +`tokenizers`, the Rust library behind the `tokenizers` npm and PyPI packages, +at the same release as those packages. + +- `third_party/tokenizers` = `huggingface/tokenizers` **v0.23.2** (`88a4498a`). +- `hf_tokenizers/` is our C ABI over it: `onebit_tok_from_json`, `_encode`, + `_decode`, `_vocab_size` and `_last_error`. It is built as a static library + with `default-features = false, features = ["onig"]`; the progress bar and + the unigram-training accelerator are for training only. + `hf_tokenizers/include/hf_tokenizer.h` adds a small C++ owner, + `onebit::HfTokenizer`. +- The Rust release is pinned in `hf_tokenizers/rust-toolchain.toml` (1.98.1; + rustup honours it) and the crate graph in `hf_tokenizers/Cargo.lock` + (`cargo build --locked`). +- `bump-tokenizers.yml` moves the pin to each new release and refreshes the lock. + +There is no official C binding upstream. mlc-ai/tokenizers-cpp wraps the same +crate, but it pins 0.21.2. + +## Build and test + +```sh +cmake -B build -DONEBIT_HF_TOKENIZERS=ON -DONEBIT_TOKENIZER_ROOT=~/.config/flm/models +cmake --build build --target hf_tokenizer_test && (cd build && ctest -R hf_tokenizer) +``` + +`tests/data/tokenizers/.json` holds golden cases made with Python +`tokenizers==0.23.2` by `tools/make_tokenizer_golden.py`. The corpus mixes +English, code, numbers, whitespace runs, eight scripts, emoji with ZWJ and +chat markup. Each file names the sha256 of the tokenizer.json it came from. + +## Verified 2026-09-23 + +18 of 18 models, 17 of 17 cases each, ids and decoded text identical: +Qwen3 0.6B/1.7B/4B/8B, Qwen3-VL-4B, Qwen3.5-4B, Qwen3.6-35B-A3B, Llama 3.1-8B +and 3.2-1B/3B, Gemma3 1B/4B, Gemma4 E2B/E4B, Phi4-mini, LFM2 1.2B/2.6B and +Nanbeige4.1-3B. That covers vocabularies from 64k to 262k. The engine's own +Qwen golden (`tests/data/qwen3_tokenizer_golden.json`) also matches, 14 of 14. diff --git a/hf_tokenizers/Cargo.lock b/hf_tokenizers/Cargo.lock new file mode 100644 index 00000000..56404a86 --- /dev/null +++ b/hf_tokenizers/Cargo.lock @@ -0,0 +1,726 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "onebit_hf_tokenizers" +version = "0.1.0" +dependencies = [ + "tokenizers", +] + +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokenizers" +version = "0.23.2" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/hf_tokenizers/Cargo.toml b/hf_tokenizers/Cargo.toml new file mode 100644 index 00000000..729536b0 --- /dev/null +++ b/hf_tokenizers/Cargo.toml @@ -0,0 +1,35 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hf_tokenizers: a C ABI over Hugging Face `tokenizers` (third_party/tokenizers, +# the same release as the `tokenizers` npm/PyPI packages), so the C++ engine can +# load any model's tokenizer.json (docs/tokenizers.md). +[package] +name = "onebit_hf_tokenizers" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[lib] +crate-type = ["staticlib"] + +[dependencies] +tokenizers = { path = "../third_party/tokenizers/tokenizers", default-features = false, features = ["onig"] } + +[profile.release] +lto = true +codegen-units = 1 +panic = "abort" diff --git a/hf_tokenizers/include/hf_tokenizer.h b/hf_tokenizers/include/hf_tokenizer.h new file mode 100644 index 00000000..46c0bc65 --- /dev/null +++ b/hf_tokenizers/include/hf_tokenizer.h @@ -0,0 +1,98 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Hugging Face `tokenizers` for the engine (docs/tokenizers.md): the C ABI of +// hf_tokenizers/ plus a small C++ owner. Loads any model's tokenizer.json. +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct OnebitTokenizer OnebitTokenizer; +OnebitTokenizer* onebit_tok_from_json(const uint8_t* json, size_t len); +void onebit_tok_free(OnebitTokenizer* tok); +int64_t onebit_tok_encode(const OnebitTokenizer* tok, const uint8_t* text, size_t len, bool add_special, + uint32_t* out, size_t cap); +int64_t onebit_tok_decode(const OnebitTokenizer* tok, const uint32_t* ids, size_t n, bool skip_special, + uint8_t* out, size_t cap); +int64_t onebit_tok_vocab_size(const OnebitTokenizer* tok); +const char* onebit_tok_last_error(void); + +#ifdef __cplusplus +} + +#include +#include +#include +#include +#include +#include + +namespace onebit { + +class HfTokenizer { +public: + explicit HfTokenizer(const std::string& tokenizer_json_path) { + std::ifstream f(tokenizer_json_path, std::ios::binary); + if (!f) throw std::runtime_error("cannot open " + tokenizer_json_path); + const std::string json{std::istreambuf_iterator(f), {}}; + tok_.reset(onebit_tok_from_json(reinterpret_cast(json.data()), json.size())); + if (!tok_) throw std::runtime_error(tokenizer_json_path + ": " + onebit_tok_last_error()); + } + + std::vector encode(const std::string& text, bool add_special = false) const { + std::vector ids(text.size() + 8); + auto n = onebit_tok_encode(tok_.get(), reinterpret_cast(text.data()), text.size(), + add_special, ids.data(), ids.size()); + if (n < 0) throw std::runtime_error(onebit_tok_last_error()); + if (size_t(n) > ids.size()) { + ids.resize(size_t(n)); + n = onebit_tok_encode(tok_.get(), reinterpret_cast(text.data()), text.size(), + add_special, ids.data(), ids.size()); + } + ids.resize(size_t(n)); + return ids; + } + + std::string decode(const std::vector& ids, bool skip_special = false) const { + std::string s(ids.size() * 8 + 16, '\0'); + auto n = onebit_tok_decode(tok_.get(), ids.data(), ids.size(), skip_special, + reinterpret_cast(s.data()), s.size()); + if (n < 0) throw std::runtime_error(onebit_tok_last_error()); + if (size_t(n) > s.size()) { + s.resize(size_t(n)); + n = onebit_tok_decode(tok_.get(), ids.data(), ids.size(), skip_special, + reinterpret_cast(s.data()), s.size()); + } + s.resize(size_t(n)); + return s; + } + + size_t vocab_size() const { return size_t(onebit_tok_vocab_size(tok_.get())); } + +private: + struct Free { + void operator()(OnebitTokenizer* t) const { onebit_tok_free(t); } + }; + std::unique_ptr tok_; +}; + +} // namespace onebit +#endif diff --git a/hf_tokenizers/rust-toolchain.toml b/hf_tokenizers/rust-toolchain.toml new file mode 100644 index 00000000..b0da1ed6 --- /dev/null +++ b/hf_tokenizers/rust-toolchain.toml @@ -0,0 +1,19 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The Rust toolchain hf_tokenizers builds with (scripts/build-tokenizers.sh). +[toolchain] +channel = "1.98.1" +profile = "minimal" diff --git a/hf_tokenizers/src/lib.rs b/hf_tokenizers/src/lib.rs new file mode 100644 index 00000000..faf2f9c8 --- /dev/null +++ b/hf_tokenizers/src/lib.rs @@ -0,0 +1,136 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// C ABI over Hugging Face `tokenizers` (include/hf_tokenizer.h). Every function +// is panic-free at the boundary: errors come back as NULL / -1, and the message +// is kept for onebit_tok_last_error(). +use std::cell::RefCell; +use std::ffi::{c_char, CString}; +use std::slice; + +use tokenizers::Tokenizer; + +thread_local! { + static LAST_ERROR: RefCell = RefCell::new(CString::default()); +} + +fn set_error(msg: String) { + LAST_ERROR.with(|e| *e.borrow_mut() = CString::new(msg.replace('\0', " ")).unwrap_or_default()); +} + +/// Parses a tokenizer.json held in memory. Returns NULL on error. +#[no_mangle] +pub unsafe extern "C" fn onebit_tok_from_json(json: *const u8, len: usize) -> *mut Tokenizer { + if json.is_null() { + set_error("null json".into()); + return std::ptr::null_mut(); + } + let bytes = slice::from_raw_parts(json, len); + match Tokenizer::from_bytes(bytes) { + Ok(t) => Box::into_raw(Box::new(t)), + Err(e) => { + set_error(e.to_string()); + std::ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn onebit_tok_free(tok: *mut Tokenizer) { + if !tok.is_null() { + drop(Box::from_raw(tok)); + } +} + +/// Encodes UTF-8 text. Writes up to `cap` ids to `out` and returns the total +/// number of ids (call again with a bigger buffer if it exceeds `cap`), or -1. +#[no_mangle] +pub unsafe extern "C" fn onebit_tok_encode( + tok: *const Tokenizer, + text: *const u8, + len: usize, + add_special: bool, + out: *mut u32, + cap: usize, +) -> i64 { + if tok.is_null() || (text.is_null() && len > 0) { + set_error("null argument".into()); + return -1; + } + let s = match std::str::from_utf8(if len == 0 { &[] } else { slice::from_raw_parts(text, len) }) { + Ok(s) => s, + Err(e) => { + set_error(e.to_string()); + return -1; + } + }; + match (*tok).encode(s, add_special) { + Ok(enc) => { + let ids = enc.get_ids(); + if !out.is_null() { + let n = ids.len().min(cap); + std::ptr::copy_nonoverlapping(ids.as_ptr(), out, n); + } + ids.len() as i64 + } + Err(e) => { + set_error(e.to_string()); + -1 + } + } +} + +/// Decodes ids to UTF-8. Writes up to `cap` bytes (not NUL-terminated) and +/// returns the total byte length, or -1. +#[no_mangle] +pub unsafe extern "C" fn onebit_tok_decode( + tok: *const Tokenizer, + ids: *const u32, + n: usize, + skip_special: bool, + out: *mut u8, + cap: usize, +) -> i64 { + if tok.is_null() || (ids.is_null() && n > 0) { + set_error("null argument".into()); + return -1; + } + let v = if n == 0 { &[][..] } else { slice::from_raw_parts(ids, n) }; + match (*tok).decode(v, skip_special) { + Ok(s) => { + if !out.is_null() { + let m = s.len().min(cap); + std::ptr::copy_nonoverlapping(s.as_ptr(), out, m); + } + s.len() as i64 + } + Err(e) => { + set_error(e.to_string()); + -1 + } + } +} + +/// Vocabulary size including added tokens. +#[no_mangle] +pub unsafe extern "C" fn onebit_tok_vocab_size(tok: *const Tokenizer) -> i64 { + if tok.is_null() { -1 } else { (*tok).get_vocab_size(true) as i64 } +} + +/// The last error on this thread, valid until the next failing call. +#[no_mangle] +pub extern "C" fn onebit_tok_last_error() -> *const c_char { + LAST_ERROR.with(|e| e.borrow().as_ptr()) +} diff --git a/tests/data/tokenizers/Gemma3-1B-NPU2.json b/tests/data/tokenizers/Gemma3-1B-NPU2.json new file mode 100644 index 00000000..39380255 --- /dev/null +++ b/tests/data/tokenizers/Gemma3-1B-NPU2.json @@ -0,0 +1,329 @@ +{ + "source": "tokenizers 0.23.2, Gemma3-1B-NPU2 tokenizer.json sha256 4667f2089529", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 818, + 5279, + 529, + 7001, + 563 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9259, + 236764, + 1902, + 236888, + 2088, + 236789, + 236751, + 625, + 1771, + 236881 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 138, + 26016, + 532, + 139, + 3859, + 139, + 35220, + 255968, + 624, + 255968, + 39218, + 108, + 208697 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 37091, + 236787, + 236743, + 236800, + 236761, + 236770, + 236812, + 236770, + 236810, + 236819, + 236764, + 236743, + 236770, + 236764, + 236771, + 236771, + 236771, + 236764, + 236771, + 236771, + 236771, + 532, + 236743, + 236778, + 236771, + 236778, + 236825, + 236772, + 236771, + 236819, + 236772, + 236778, + 236800, + 236774, + 236770, + 236832, + 236787, + 236800, + 236771, + 236787, + 236771, + 236771, + 236953 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 2063, + 517, + 236769, + 236781, + 1473, + 107, + 140, + 2060, + 1123, + 1018, + 236778, + 138, + 236865, + 6281, + 107 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 584, + 568, + 236746, + 2843, + 518, + 2732, + 505, + 6605, + 513, + 236768, + 642, + 994, + 5325, + 1480, + 2074, + 236769, + 236746, + 236764, + 518, + 626, + 682 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 238194, + 236749, + 238527, + 236755, + 55515, + 236859, + 120362, + 33443, + 118515 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 94951, + 236945, + 95830, + 237032, + 201637 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 116130, + 236764, + 6815, + 22633, + 236881 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 236873, + 150345, + 107602, + 17193 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 226767, + 19975 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 246143, + 242015, + 64334, + 103712, + 242713, + 532, + 1491, + 236824, + 236863, + 236743, + 243767, + 237243, + 244862 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 236777, + 236789, + 236757, + 692, + 236789, + 500, + 901, + 236789, + 560, + 668, + 236789, + 236753, + 1304, + 236789, + 859, + 625, + 236789, + 236751 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 236820, + 236909, + 548, + 236779, + 3041, + 111038, + 2364, + 107, + 3689, + 563, + 236743, + 236778, + 236862, + 236778, + 145912, + 236909, + 548, + 236779, + 643, + 111038 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 236743 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 50354 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Gemma3-4B-NPU2.json b/tests/data/tokenizers/Gemma3-4B-NPU2.json new file mode 100644 index 00000000..bea84f5c --- /dev/null +++ b/tests/data/tokenizers/Gemma3-4B-NPU2.json @@ -0,0 +1,329 @@ +{ + "source": "tokenizers 0.23.2, Gemma3-4B-NPU2 tokenizer.json sha256 4667f2089529", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 818, + 5279, + 529, + 7001, + 563 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9259, + 236764, + 1902, + 236888, + 2088, + 236789, + 236751, + 625, + 1771, + 236881 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 138, + 26016, + 532, + 139, + 3859, + 139, + 35220, + 255968, + 624, + 255968, + 39218, + 108, + 208697 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 37091, + 236787, + 236743, + 236800, + 236761, + 236770, + 236812, + 236770, + 236810, + 236819, + 236764, + 236743, + 236770, + 236764, + 236771, + 236771, + 236771, + 236764, + 236771, + 236771, + 236771, + 532, + 236743, + 236778, + 236771, + 236778, + 236825, + 236772, + 236771, + 236819, + 236772, + 236778, + 236800, + 236774, + 236770, + 236832, + 236787, + 236800, + 236771, + 236787, + 236771, + 236771, + 236953 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 2063, + 517, + 236769, + 236781, + 1473, + 107, + 140, + 2060, + 1123, + 1018, + 236778, + 138, + 236865, + 6281, + 107 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 584, + 568, + 236746, + 2843, + 518, + 2732, + 505, + 6605, + 513, + 236768, + 642, + 994, + 5325, + 1480, + 2074, + 236769, + 236746, + 236764, + 518, + 626, + 682 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 238194, + 236749, + 238527, + 236755, + 55515, + 236859, + 120362, + 33443, + 118515 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 94951, + 236945, + 95830, + 237032, + 201637 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 116130, + 236764, + 6815, + 22633, + 236881 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 236873, + 150345, + 107602, + 17193 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 226767, + 19975 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 246143, + 242015, + 64334, + 103712, + 242713, + 532, + 1491, + 236824, + 236863, + 236743, + 243767, + 237243, + 244862 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 236777, + 236789, + 236757, + 692, + 236789, + 500, + 901, + 236789, + 560, + 668, + 236789, + 236753, + 1304, + 236789, + 859, + 625, + 236789, + 236751 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 236820, + 236909, + 548, + 236779, + 3041, + 111038, + 2364, + 107, + 3689, + 563, + 236743, + 236778, + 236862, + 236778, + 145912, + 236909, + 548, + 236779, + 643, + 111038 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 236743 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 50354 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Gemma4-E2B-IT-NPU2.json b/tests/data/tokenizers/Gemma4-E2B-IT-NPU2.json new file mode 100644 index 00000000..8ae2e2f5 --- /dev/null +++ b/tests/data/tokenizers/Gemma4-E2B-IT-NPU2.json @@ -0,0 +1,329 @@ +{ + "source": "tokenizers 0.23.2, Gemma4-E2B-IT-NPU2 tokenizer.json sha256 cc8d3a0ce364", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 818, + 5279, + 529, + 7001, + 563 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9259, + 236764, + 1902, + 236888, + 2088, + 236789, + 236751, + 625, + 1771, + 236881 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 138, + 26016, + 532, + 139, + 3859, + 139, + 35220, + 255968, + 624, + 255968, + 39218, + 108, + 208697 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 37091, + 236787, + 236743, + 236800, + 236761, + 236770, + 236812, + 236770, + 236810, + 236819, + 236764, + 236743, + 236770, + 236764, + 236771, + 236771, + 236771, + 236764, + 236771, + 236771, + 236771, + 532, + 236743, + 236778, + 236771, + 236778, + 236825, + 236772, + 236771, + 236819, + 236772, + 236778, + 236800, + 236774, + 236770, + 236832, + 236787, + 236800, + 236771, + 236787, + 236771, + 236771, + 236953 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 2063, + 517, + 236769, + 236781, + 1473, + 107, + 140, + 2060, + 1123, + 1018, + 236778, + 138, + 236865, + 6281, + 107 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 584, + 568, + 236746, + 2843, + 518, + 2732, + 505, + 6605, + 513, + 236768, + 642, + 994, + 5325, + 1480, + 2074, + 236769, + 236746, + 236764, + 518, + 626, + 682 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 238194, + 236749, + 238527, + 236755, + 55515, + 236859, + 120362, + 33443, + 118515 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 94951, + 236945, + 95830, + 237032, + 201637 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 116130, + 236764, + 6815, + 22633, + 236881 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 236873, + 150345, + 107602, + 17193 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 226767, + 19975 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 246143, + 242015, + 64334, + 103712, + 242713, + 532, + 1491, + 236824, + 236863, + 236743, + 243767, + 237243, + 244862 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 236777, + 236789, + 236757, + 692, + 236789, + 500, + 901, + 236789, + 560, + 668, + 236789, + 236753, + 1304, + 236789, + 859, + 625, + 236789, + 236751 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 236820, + 236909, + 548, + 236779, + 3041, + 111038, + 2364, + 107, + 3689, + 563, + 236743, + 236778, + 236862, + 236778, + 145912, + 236909, + 548, + 236779, + 643, + 111038 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 236743 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 50354 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json b/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json new file mode 100644 index 00000000..0ba1256b --- /dev/null +++ b/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json @@ -0,0 +1,329 @@ +{ + "source": "tokenizers 0.23.2, Gemma4-E4B-IT-NPU2 tokenizer.json sha256 cc8d3a0ce364", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 818, + 5279, + 529, + 7001, + 563 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9259, + 236764, + 1902, + 236888, + 2088, + 236789, + 236751, + 625, + 1771, + 236881 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 138, + 26016, + 532, + 139, + 3859, + 139, + 35220, + 255968, + 624, + 255968, + 39218, + 108, + 208697 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 37091, + 236787, + 236743, + 236800, + 236761, + 236770, + 236812, + 236770, + 236810, + 236819, + 236764, + 236743, + 236770, + 236764, + 236771, + 236771, + 236771, + 236764, + 236771, + 236771, + 236771, + 532, + 236743, + 236778, + 236771, + 236778, + 236825, + 236772, + 236771, + 236819, + 236772, + 236778, + 236800, + 236774, + 236770, + 236832, + 236787, + 236800, + 236771, + 236787, + 236771, + 236771, + 236953 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 2063, + 517, + 236769, + 236781, + 1473, + 107, + 140, + 2060, + 1123, + 1018, + 236778, + 138, + 236865, + 6281, + 107 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 584, + 568, + 236746, + 2843, + 518, + 2732, + 505, + 6605, + 513, + 236768, + 642, + 994, + 5325, + 1480, + 2074, + 236769, + 236746, + 236764, + 518, + 626, + 682 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 238194, + 236749, + 238527, + 236755, + 55515, + 236859, + 120362, + 33443, + 118515 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 94951, + 236945, + 95830, + 237032, + 201637 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 116130, + 236764, + 6815, + 22633, + 236881 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 236873, + 150345, + 107602, + 17193 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 226767, + 19975 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 246143, + 242015, + 64334, + 103712, + 242713, + 532, + 1491, + 236824, + 236863, + 236743, + 243767, + 237243, + 244862 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 236777, + 236789, + 236757, + 692, + 236789, + 500, + 901, + 236789, + 560, + 668, + 236789, + 236753, + 1304, + 236789, + 859, + 625, + 236789, + 236751 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 236820, + 236909, + 548, + 236779, + 3041, + 111038, + 2364, + 107, + 3689, + 563, + 236743, + 236778, + 236862, + 236778, + 145912, + 236909, + 548, + 236779, + 643, + 111038 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 236743 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 128820, + 50354 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/LFM2-1.2B-NPU2.json b/tests/data/tokenizers/LFM2-1.2B-NPU2.json new file mode 100644 index 00000000..24e22e96 --- /dev/null +++ b/tests/data/tokenizers/LFM2-1.2B-NPU2.json @@ -0,0 +1,376 @@ +{ + "source": "tokenizers 0.23.2, LFM2-1.2B-NPU2 tokenizer.json sha256 98cff83b4f6d", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 1098, + 5706, + 803, + 4481, + 856 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 36309, + 521, + 2031, + 510, + 2213, + 1090, + 936, + 3529, + 540 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 730, + 5845, + 810, + 767, + 9212, + 767, + 12914, + 707, + 905, + 15153, + 7954, + 509, + 3514, + 10890 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 555, + 20661, + 535, + 730, + 528, + 523, + 13888, + 5599, + 521, + 730, + 526, + 521, + 1873, + 521, + 1873, + 810, + 730, + 1718, + 531, + 522, + 4905, + 522, + 1968, + 561, + 1417, + 535, + 1664, + 535, + 1056, + 567 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 3663, + 792, + 6780, + 5174, + 861, + 1789, + 3053, + 1463, + 527, + 730, + 2679, + 9054, + 708 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 888, + 843, + 574, + 5604, + 794, + 6331, + 781, + 26116, + 778, + 518, + 1196, + 1789, + 9877, + 2140, + 8708, + 13792, + 521, + 794, + 4032, + 2480 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 24184, + 587, + 6838, + 576, + 10905, + 860, + 2116, + 6838, + 1124, + 35499, + 7490, + 51521 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 62506, + 1084, + 4374, + 4459, + 7133, + 1329, + 8141, + 605, + 12995 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 7245, + 13328, + 521, + 8224, + 49795, + 540 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 18381, + 2763, + 2071, + 978, + 10331, + 2040, + 20058 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 6355, + 611, + 6355, + 616, + 6355, + 626, + 34801, + 607, + 50008, + 17903, + 609, + 14860, + 733, + 6355, + 611, + 61840, + 617, + 30170 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 13421, + 607, + 754, + 13421, + 758, + 732, + 1355, + 588, + 3966, + 23805, + 749, + 745, + 13421, + 747, + 631, + 810, + 1396, + 564, + 551, + 23805, + 749, + 612, + 844, + 745, + 13421, + 750, + 629 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 550, + 6217, + 1023, + 4644, + 1186, + 6402, + 950, + 6351, + 1689, + 6016, + 936, + 1090 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 6, + 6423, + 708, + 3493, + 856, + 730, + 527, + 520, + 527, + 540, + 7 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 730 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/LFM2-2.6B-NPU2.json b/tests/data/tokenizers/LFM2-2.6B-NPU2.json new file mode 100644 index 00000000..571ccccc --- /dev/null +++ b/tests/data/tokenizers/LFM2-2.6B-NPU2.json @@ -0,0 +1,376 @@ +{ + "source": "tokenizers 0.23.2, LFM2-2.6B-NPU2 tokenizer.json sha256 98cff83b4f6d", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 1098, + 5706, + 803, + 4481, + 856 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 36309, + 521, + 2031, + 510, + 2213, + 1090, + 936, + 3529, + 540 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 730, + 5845, + 810, + 767, + 9212, + 767, + 12914, + 707, + 905, + 15153, + 7954, + 509, + 3514, + 10890 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 555, + 20661, + 535, + 730, + 528, + 523, + 13888, + 5599, + 521, + 730, + 526, + 521, + 1873, + 521, + 1873, + 810, + 730, + 1718, + 531, + 522, + 4905, + 522, + 1968, + 561, + 1417, + 535, + 1664, + 535, + 1056, + 567 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 3663, + 792, + 6780, + 5174, + 861, + 1789, + 3053, + 1463, + 527, + 730, + 2679, + 9054, + 708 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 888, + 843, + 574, + 5604, + 794, + 6331, + 781, + 26116, + 778, + 518, + 1196, + 1789, + 9877, + 2140, + 8708, + 13792, + 521, + 794, + 4032, + 2480 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 24184, + 587, + 6838, + 576, + 10905, + 860, + 2116, + 6838, + 1124, + 35499, + 7490, + 51521 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 62506, + 1084, + 4374, + 4459, + 7133, + 1329, + 8141, + 605, + 12995 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 7245, + 13328, + 521, + 8224, + 49795, + 540 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 18381, + 2763, + 2071, + 978, + 10331, + 2040, + 20058 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 6355, + 611, + 6355, + 616, + 6355, + 626, + 34801, + 607, + 50008, + 17903, + 609, + 14860, + 733, + 6355, + 611, + 61840, + 617, + 30170 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 13421, + 607, + 754, + 13421, + 758, + 732, + 1355, + 588, + 3966, + 23805, + 749, + 745, + 13421, + 747, + 631, + 810, + 1396, + 564, + 551, + 23805, + 749, + 612, + 844, + 745, + 13421, + 750, + 629 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 550, + 6217, + 1023, + 4644, + 1186, + 6402, + 950, + 6351, + 1689, + 6016, + 936, + 1090 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 6, + 6423, + 708, + 3493, + 856, + 730, + 527, + 520, + 527, + 540, + 7 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 730 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738, + 44738 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Llama-3.1-8B-NPU2.json b/tests/data/tokenizers/Llama-3.1-8B-NPU2.json new file mode 100644 index 00000000..9d06f2eb --- /dev/null +++ b/tests/data/tokenizers/Llama-3.1-8B-NPU2.json @@ -0,0 +1,332 @@ +{ + "source": "tokenizers 0.23.2, Llama-3.1-8B-NPU2 tokenizer.json sha256 79e3e522635f", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 791, + 6864, + 315, + 9822, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9906, + 11, + 1917, + 0, + 2650, + 596, + 433, + 2133, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6522, + 323, + 256, + 9358, + 256, + 12908, + 53577, + 3324, + 3518, + 271, + 943, + 8128 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 28336, + 25, + 220, + 18, + 13, + 9335, + 2946, + 11, + 220, + 16, + 11, + 931, + 11, + 931, + 323, + 220, + 2366, + 21, + 12, + 2545, + 12, + 1419, + 51, + 1114, + 25, + 966, + 25, + 410, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 755, + 282, + 2120, + 997, + 262, + 471, + 865, + 334, + 17, + 220, + 674, + 9518, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 976, + 293, + 1024, + 272, + 2717, + 294, + 8, + 314, + 471, + 1487, + 487, + 2880, + 2948, + 11, + 293, + 1237, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 53591, + 77, + 38672, + 66, + 3029, + 67, + 978, + 95980, + 588, + 53050, + 9517, + 1264, + 978 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 102433, + 102158, + 16144, + 57933, + 62903, + 71634, + 19732, + 115953, + 19113 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 54745, + 28089, + 8341, + 11, + 52770, + 123936, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 101244, + 30925, + 103645, + 100700, + 24102, + 101952 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 61196, + 88344, + 79468, + 100365, + 35470, + 100291, + 101391, + 100322, + 24810 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 9468, + 97, + 244, + 9468, + 248, + 222, + 43465, + 62904, + 235, + 9468, + 237, + 121, + 323, + 1901, + 54, + 41, + 62904, + 102, + 102470, + 93273, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2846, + 584, + 2351, + 814, + 3077, + 568, + 4265, + 1364, + 3358, + 433, + 596 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 27, + 91, + 318, + 5011, + 91, + 29, + 882, + 198, + 3923, + 374, + 220, + 17, + 10, + 17, + 76514, + 91, + 318, + 6345, + 91, + 29 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 29558 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Llama-3.2-1B-NPU2.json b/tests/data/tokenizers/Llama-3.2-1B-NPU2.json new file mode 100644 index 00000000..b2565ea7 --- /dev/null +++ b/tests/data/tokenizers/Llama-3.2-1B-NPU2.json @@ -0,0 +1,332 @@ +{ + "source": "tokenizers 0.23.2, Llama-3.2-1B-NPU2 tokenizer.json sha256 79e3e522635f", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 791, + 6864, + 315, + 9822, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9906, + 11, + 1917, + 0, + 2650, + 596, + 433, + 2133, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6522, + 323, + 256, + 9358, + 256, + 12908, + 53577, + 3324, + 3518, + 271, + 943, + 8128 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 28336, + 25, + 220, + 18, + 13, + 9335, + 2946, + 11, + 220, + 16, + 11, + 931, + 11, + 931, + 323, + 220, + 2366, + 21, + 12, + 2545, + 12, + 1419, + 51, + 1114, + 25, + 966, + 25, + 410, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 755, + 282, + 2120, + 997, + 262, + 471, + 865, + 334, + 17, + 220, + 674, + 9518, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 976, + 293, + 1024, + 272, + 2717, + 294, + 8, + 314, + 471, + 1487, + 487, + 2880, + 2948, + 11, + 293, + 1237, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 53591, + 77, + 38672, + 66, + 3029, + 67, + 978, + 95980, + 588, + 53050, + 9517, + 1264, + 978 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 102433, + 102158, + 16144, + 57933, + 62903, + 71634, + 19732, + 115953, + 19113 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 54745, + 28089, + 8341, + 11, + 52770, + 123936, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 101244, + 30925, + 103645, + 100700, + 24102, + 101952 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 61196, + 88344, + 79468, + 100365, + 35470, + 100291, + 101391, + 100322, + 24810 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 9468, + 97, + 244, + 9468, + 248, + 222, + 43465, + 62904, + 235, + 9468, + 237, + 121, + 323, + 1901, + 54, + 41, + 62904, + 102, + 102470, + 93273, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2846, + 584, + 2351, + 814, + 3077, + 568, + 4265, + 1364, + 3358, + 433, + 596 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 27, + 91, + 318, + 5011, + 91, + 29, + 882, + 198, + 3923, + 374, + 220, + 17, + 10, + 17, + 76514, + 91, + 318, + 6345, + 91, + 29 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 29558 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Llama-3.2-3B-NPU2.json b/tests/data/tokenizers/Llama-3.2-3B-NPU2.json new file mode 100644 index 00000000..69c9a4db --- /dev/null +++ b/tests/data/tokenizers/Llama-3.2-3B-NPU2.json @@ -0,0 +1,332 @@ +{ + "source": "tokenizers 0.23.2, Llama-3.2-3B-NPU2 tokenizer.json sha256 79e3e522635f", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 791, + 6864, + 315, + 9822, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9906, + 11, + 1917, + 0, + 2650, + 596, + 433, + 2133, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6522, + 323, + 256, + 9358, + 256, + 12908, + 53577, + 3324, + 3518, + 271, + 943, + 8128 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 28336, + 25, + 220, + 18, + 13, + 9335, + 2946, + 11, + 220, + 16, + 11, + 931, + 11, + 931, + 323, + 220, + 2366, + 21, + 12, + 2545, + 12, + 1419, + 51, + 1114, + 25, + 966, + 25, + 410, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 755, + 282, + 2120, + 997, + 262, + 471, + 865, + 334, + 17, + 220, + 674, + 9518, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 976, + 293, + 1024, + 272, + 2717, + 294, + 8, + 314, + 471, + 1487, + 487, + 2880, + 2948, + 11, + 293, + 1237, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 53591, + 77, + 38672, + 66, + 3029, + 67, + 978, + 95980, + 588, + 53050, + 9517, + 1264, + 978 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 102433, + 102158, + 16144, + 57933, + 62903, + 71634, + 19732, + 115953, + 19113 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 54745, + 28089, + 8341, + 11, + 52770, + 123936, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 101244, + 30925, + 103645, + 100700, + 24102, + 101952 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 61196, + 88344, + 79468, + 100365, + 35470, + 100291, + 101391, + 100322, + 24810 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 9468, + 97, + 244, + 9468, + 248, + 222, + 43465, + 62904, + 235, + 9468, + 237, + 121, + 323, + 1901, + 54, + 41, + 62904, + 102, + 102470, + 93273, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2846, + 584, + 2351, + 814, + 3077, + 568, + 4265, + 1364, + 3358, + 433, + 596 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 27, + 91, + 318, + 5011, + 91, + 29, + 882, + 198, + 3923, + 374, + 220, + 17, + 10, + 17, + 76514, + 91, + 318, + 6345, + 91, + 29 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 70540, + 29558 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Nanbeige4.1-3B-NPU2.json b/tests/data/tokenizers/Nanbeige4.1-3B-NPU2.json new file mode 100644 index 00000000..9958d90a --- /dev/null +++ b/tests/data/tokenizers/Nanbeige4.1-3B-NPU2.json @@ -0,0 +1,361 @@ +{ + "source": "tokenizers 0.23.2, Nanbeige4.1-3B-NPU2 tokenizer.json sha256 1d8f03269101", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 363, + 5463, + 290, + 7914, + 322 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 23877, + 152363, + 1427, + 152518, + 1064, + 152402, + 152350, + 388, + 2087, + 152470 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 259, + 4646, + 297, + 259, + 6960, + 259, + 8347, + 12, + 407, + 12, + 46403, + 13, + 13, + 2071, + 7707 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 32050, + 152382, + 152343, + 152385, + 152361, + 152367, + 152395, + 152367, + 152394, + 152398, + 152363, + 152343, + 152367, + 152363, + 152368, + 152368, + 152368, + 152363, + 152368, + 152368, + 152368, + 297, + 152343, + 152375, + 152368, + 152375, + 152408, + 152369, + 152368, + 152398, + 152369, + 152375, + 152385, + 152376, + 152367, + 152413, + 152382, + 152385, + 152368, + 152382, + 152368, + 152368, + 152634 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 862, + 283, + 152374, + 152381, + 1123, + 13, + 264, + 1590, + 1056, + 454, + 152375, + 259, + 152458, + 6457, + 13 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 622, + 326, + 152346, + 3198, + 288, + 3599, + 273, + 7840, + 292, + 152373, + 500, + 926, + 4616, + 1048, + 3365, + 152374, + 152346, + 152363, + 288, + 446, + 603 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 31243, + 152349, + 154564, + 152355, + 133060, + 152734, + 77485, + 84959, + 12351, + 143404 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 35632, + 156612, + 154064, + 156163, + 156685, + 67037, + 154740, + 158211, + 152815 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 45969, + 28727, + 152363, + 22816, + 96278, + 152470 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 45105, + 154968, + 156300, + 155335, + 154233, + 50173, + 21899, + 155716, + 21899, + 154936 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 152343, + 156578, + 156761, + 156686, + 155943, + 156551, + 156421, + 152343, + 157404, + 157772, + 156578, + 156540, + 156988, + 155798 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 152343, + 243, + 162, + 167, + 153, + 159100, + 106765, + 152343, + 158080, + 243, + 162, + 146, + 192, + 297, + 1068, + 152418, + 152436, + 152343, + 243, + 162, + 148, + 172, + 155303, + 243, + 162, + 149, + 190 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 324, + 152402, + 152357, + 457, + 152402, + 265, + 623, + 152402, + 321, + 430, + 152402, + 152354, + 1097, + 152402, + 639, + 388, + 152402, + 152350 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 166100, + 2028, + 13, + 3512, + 322, + 152343, + 152375, + 152461, + 152375, + 152470, + 166101 + ], + "decoded": "<|im_start|> user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 259 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 261, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 90405, + 48367 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Phi4-mini-Instruct-NPU2.json b/tests/data/tokenizers/Phi4-mini-Instruct-NPU2.json new file mode 100644 index 00000000..1a5cc8d8 --- /dev/null +++ b/tests/data/tokenizers/Phi4-mini-Instruct-NPU2.json @@ -0,0 +1,311 @@ +{ + "source": "tokenizers 0.23.2, Phi4-mini-Instruct-NPU2 tokenizer.json sha256 382cc235b56c", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 976, + 9029, + 328, + 10128, + 382 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 13225, + 11, + 2375, + 0, + 3253, + 885, + 480, + 2966, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 8117, + 326, + 256, + 12694, + 256, + 18608, + 128995, + 6264, + 6071, + 279, + 1389, + 10105 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 31274, + 25, + 220, + 18, + 13, + 16926, + 4621, + 11, + 220, + 16, + 11, + 1302, + 11, + 1302, + 326, + 220, + 1323, + 21, + 12, + 3114, + 12, + 1860, + 51, + 1422, + 25, + 1130, + 25, + 504, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 1314, + 285, + 4061, + 1883, + 271, + 622, + 1215, + 410, + 17, + 220, + 1069, + 13749, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 366, + 350, + 64, + 1666, + 287, + 1725, + 274, + 5017, + 272, + 8, + 354, + 622, + 3081, + 742, + 3228, + 6271, + 11, + 287, + 2245, + 388 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 8858, + 77, + 191375, + 43369, + 377, + 153475, + 737, + 30469, + 140184 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 9048, + 40909, + 3385, + 16056, + 18368, + 38236, + 5330, + 183591, + 8134 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 23881, + 131903, + 11, + 6220, + 78857, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 158894, + 26537, + 101462, + 12773 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 998, + 1637, + 14681, + 628, + 64593 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 50378, + 244, + 112927, + 222, + 74471, + 160433, + 52622, + 121, + 326, + 1489, + 54, + 41, + 61138, + 102, + 2524, + 31446, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 15390, + 15929, + 51676, + 71619, + 141340, + 4275 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 27, + 91, + 321, + 10949, + 91, + 29, + 1428, + 198, + 4827, + 382, + 220, + 17, + 10, + 17, + 190440, + 91, + 321, + 13707, + 91, + 29 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 117525, + 45037 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3-0.6B-NPU2.json b/tests/data/tokenizers/Qwen3-0.6B-NPU2.json new file mode 100644 index 00000000..4960a838 --- /dev/null +++ b/tests/data/tokenizers/Qwen3-0.6B-NPU2.json @@ -0,0 +1,333 @@ +{ + "source": "tokenizers 0.23.2, Qwen3-0.6B-NPU2 tokenizer.json sha256 aeb13307a71a", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9707, + 11, + 1879, + 0, + 2585, + 594, + 432, + 2087, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6388, + 323, + 256, + 9179, + 256, + 12621, + 52477, + 3244, + 3435, + 271, + 931, + 7969 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 27237, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 323, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 334, + 17, + 220, + 671, + 9334, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 961, + 293, + 1009, + 272, + 2651, + 294, + 8, + 314, + 470, + 1460, + 486, + 2810, + 2877, + 11, + 293, + 1215, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 52491, + 77, + 37572, + 66, + 2956, + 128505, + 94880, + 586, + 51950, + 9333, + 1242, + 963 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 101059, + 102819, + 15767, + 56833, + 61803, + 70534, + 19182, + 110007, + 18600 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 53645, + 26991, + 8178, + 11, + 51670, + 129691, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 147238, + 145836, + 42365, + 61804, + 235, + 145375, + 323, + 1863, + 54, + 41, + 61804, + 102, + 378, + 235, + 145851 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2776, + 582, + 2299, + 807, + 3003, + 566, + 4172, + 1340, + 3278, + 432, + 594 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 151644, + 872, + 198, + 3838, + 374, + 220, + 17, + 10, + 17, + 30, + 151645 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 28458 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3-1.7B-NPU2.json b/tests/data/tokenizers/Qwen3-1.7B-NPU2.json new file mode 100644 index 00000000..059e8157 --- /dev/null +++ b/tests/data/tokenizers/Qwen3-1.7B-NPU2.json @@ -0,0 +1,333 @@ +{ + "source": "tokenizers 0.23.2, Qwen3-1.7B-NPU2 tokenizer.json sha256 aeb13307a71a", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9707, + 11, + 1879, + 0, + 2585, + 594, + 432, + 2087, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6388, + 323, + 256, + 9179, + 256, + 12621, + 52477, + 3244, + 3435, + 271, + 931, + 7969 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 27237, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 323, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 334, + 17, + 220, + 671, + 9334, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 961, + 293, + 1009, + 272, + 2651, + 294, + 8, + 314, + 470, + 1460, + 486, + 2810, + 2877, + 11, + 293, + 1215, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 52491, + 77, + 37572, + 66, + 2956, + 128505, + 94880, + 586, + 51950, + 9333, + 1242, + 963 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 101059, + 102819, + 15767, + 56833, + 61803, + 70534, + 19182, + 110007, + 18600 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 53645, + 26991, + 8178, + 11, + 51670, + 129691, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 147238, + 145836, + 42365, + 61804, + 235, + 145375, + 323, + 1863, + 54, + 41, + 61804, + 102, + 378, + 235, + 145851 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2776, + 582, + 2299, + 807, + 3003, + 566, + 4172, + 1340, + 3278, + 432, + 594 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 151644, + 872, + 198, + 3838, + 374, + 220, + 17, + 10, + 17, + 30, + 151645 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 28458 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3-4B-NPU2.json b/tests/data/tokenizers/Qwen3-4B-NPU2.json new file mode 100644 index 00000000..ace5b289 --- /dev/null +++ b/tests/data/tokenizers/Qwen3-4B-NPU2.json @@ -0,0 +1,333 @@ +{ + "source": "tokenizers 0.23.2, Qwen3-4B-NPU2 tokenizer.json sha256 aeb13307a71a", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9707, + 11, + 1879, + 0, + 2585, + 594, + 432, + 2087, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6388, + 323, + 256, + 9179, + 256, + 12621, + 52477, + 3244, + 3435, + 271, + 931, + 7969 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 27237, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 323, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 334, + 17, + 220, + 671, + 9334, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 961, + 293, + 1009, + 272, + 2651, + 294, + 8, + 314, + 470, + 1460, + 486, + 2810, + 2877, + 11, + 293, + 1215, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 52491, + 77, + 37572, + 66, + 2956, + 128505, + 94880, + 586, + 51950, + 9333, + 1242, + 963 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 101059, + 102819, + 15767, + 56833, + 61803, + 70534, + 19182, + 110007, + 18600 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 53645, + 26991, + 8178, + 11, + 51670, + 129691, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 147238, + 145836, + 42365, + 61804, + 235, + 145375, + 323, + 1863, + 54, + 41, + 61804, + 102, + 378, + 235, + 145851 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2776, + 582, + 2299, + 807, + 3003, + 566, + 4172, + 1340, + 3278, + 432, + 594 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 151644, + 872, + 198, + 3838, + 374, + 220, + 17, + 10, + 17, + 30, + 151645 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 28458 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3-8B-NPU2.json b/tests/data/tokenizers/Qwen3-8B-NPU2.json new file mode 100644 index 00000000..cfe46e97 --- /dev/null +++ b/tests/data/tokenizers/Qwen3-8B-NPU2.json @@ -0,0 +1,333 @@ +{ + "source": "tokenizers 0.23.2, Qwen3-8B-NPU2 tokenizer.json sha256 aeb13307a71a", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9707, + 11, + 1879, + 0, + 2585, + 594, + 432, + 2087, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6388, + 323, + 256, + 9179, + 256, + 12621, + 52477, + 3244, + 3435, + 271, + 931, + 7969 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 27237, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 323, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 334, + 17, + 220, + 671, + 9334, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 961, + 293, + 1009, + 272, + 2651, + 294, + 8, + 314, + 470, + 1460, + 486, + 2810, + 2877, + 11, + 293, + 1215, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 52491, + 77, + 37572, + 66, + 2956, + 128505, + 94880, + 586, + 51950, + 9333, + 1242, + 963 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 101059, + 102819, + 15767, + 56833, + 61803, + 70534, + 19182, + 110007, + 18600 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 53645, + 26991, + 8178, + 11, + 51670, + 129691, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 147238, + 145836, + 42365, + 61804, + 235, + 145375, + 323, + 1863, + 54, + 41, + 61804, + 102, + 378, + 235, + 145851 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2776, + 582, + 2299, + 807, + 3003, + 566, + 4172, + 1340, + 3278, + 432, + 594 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 151644, + 872, + 198, + 3838, + 374, + 220, + 17, + 10, + 17, + 30, + 151645 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 28458 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3-VL-4B-Instruct-NPU2.json b/tests/data/tokenizers/Qwen3-VL-4B-Instruct-NPU2.json new file mode 100644 index 00000000..560b2586 --- /dev/null +++ b/tests/data/tokenizers/Qwen3-VL-4B-Instruct-NPU2.json @@ -0,0 +1,333 @@ +{ + "source": "tokenizers 0.23.2, Qwen3-VL-4B-Instruct-NPU2 tokenizer.json sha256 aeb13307a71a", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9707, + 11, + 1879, + 0, + 2585, + 594, + 432, + 2087, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6388, + 323, + 256, + 9179, + 256, + 12621, + 52477, + 3244, + 3435, + 271, + 931, + 7969 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 27237, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 323, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 334, + 17, + 220, + 671, + 9334, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 333, + 320, + 64, + 961, + 293, + 1009, + 272, + 2651, + 294, + 8, + 314, + 470, + 1460, + 486, + 2810, + 2877, + 11, + 293, + 1215, + 335 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 52491, + 77, + 37572, + 66, + 2956, + 128505, + 94880, + 586, + 51950, + 9333, + 1242, + 963 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 101059, + 102819, + 15767, + 56833, + 61803, + 70534, + 19182, + 110007, + 18600 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 53645, + 26991, + 8178, + 11, + 51670, + 129691, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 147238, + 145836, + 42365, + 61804, + 235, + 145375, + 323, + 1863, + 54, + 41, + 61804, + 102, + 378, + 235, + 145851 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2776, + 582, + 2299, + 807, + 3003, + 566, + 4172, + 1340, + 3278, + 432, + 594 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 151644, + 872, + 198, + 3838, + 374, + 220, + 17, + 10, + 17, + 30, + 151645 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 69440, + 28458 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3.5-4B-NPU2.json b/tests/data/tokenizers/Qwen3.5-4B-NPU2.json new file mode 100644 index 00000000..23409272 --- /dev/null +++ b/tests/data/tokenizers/Qwen3.5-4B-NPU2.json @@ -0,0 +1,328 @@ +{ + "source": "tokenizers 0.23.2, Qwen3.5-4B-NPU2 tokenizer.json sha256 5f9e4d4901a9", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 760, + 6511, + 314, + 9338, + 369 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9419, + 11, + 1814, + 0, + 2500, + 579, + 424, + 2019, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6187, + 321, + 256, + 8906, + 256, + 12258, + 50711, + 3139, + 3325, + 271, + 902, + 7718 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 26359, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 321, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 727, + 281, + 2007, + 1590, + 198, + 262, + 460, + 830, + 332, + 17, + 220, + 653, + 9057, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 331, + 318, + 64, + 931, + 292, + 976, + 272, + 2564, + 293, + 8, + 313, + 460, + 1407, + 476, + 2721, + 2784, + 11, + 292, + 1174, + 333 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 50724, + 77, + 245787, + 2862, + 164497, + 91603, + 571, + 50203, + 238976 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 247359, + 15303, + 210342, + 18602, + 101306, + 96356 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 51844, + 26125, + 7921, + 11, + 150914, + 170220, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 148739, + 28850, + 150027, + 182946, + 149650 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 58069, + 84237, + 150104, + 153348, + 184642, + 235886 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 9008, + 97, + 244, + 9008, + 248, + 222, + 40958, + 59720, + 235, + 9008, + 237, + 121, + 321, + 1799, + 54, + 41, + 59720, + 102, + 373, + 235, + 88995, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2688, + 567, + 2224, + 781, + 2908, + 551, + 4035, + 1292, + 3172, + 424, + 579 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 248045, + 846, + 198, + 3710, + 369, + 220, + 17, + 10, + 17, + 30, + 248046 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 27528 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/data/tokenizers/Qwen3.6-35B-A3B-NPU2.json b/tests/data/tokenizers/Qwen3.6-35B-A3B-NPU2.json new file mode 100644 index 00000000..8b03a4a9 --- /dev/null +++ b/tests/data/tokenizers/Qwen3.6-35B-A3B-NPU2.json @@ -0,0 +1,328 @@ +{ + "source": "tokenizers 0.23.2, Qwen3.6-35B-A3B-NPU2 tokenizer.json sha256 5f9e4d4901a9", + "add_special": false, + "cases": [ + { + "text": "The capital of France is", + "ids": [ + 760, + 6511, + 314, + 9338, + 369 + ], + "decoded": "The capital of France is" + }, + { + "text": "Hello, world! How's it going?", + "ids": [ + 9419, + 11, + 1814, + 0, + 2500, + 579, + 424, + 2019, + 30 + ], + "decoded": "Hello, world! How's it going?" + }, + { + "text": " leading and inner spaces\tand\ttabs\n\nnewlines", + "ids": [ + 220, + 6187, + 321, + 256, + 8906, + 256, + 12258, + 50711, + 3139, + 3325, + 271, + 902, + 7718 + ], + "decoded": " leading and inner spaces\tand\ttabs\n\nnewlines" + }, + { + "text": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "ids": [ + 26359, + 25, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 11, + 220, + 16, + 11, + 15, + 15, + 15, + 11, + 15, + 15, + 15, + 321, + 220, + 17, + 15, + 17, + 21, + 12, + 15, + 24, + 12, + 17, + 18, + 51, + 16, + 22, + 25, + 18, + 15, + 25, + 15, + 15, + 57 + ], + "decoded": "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z" + }, + { + "text": "def f(x):\n return x**2 # square\n", + "ids": [ + 727, + 281, + 2007, + 1590, + 198, + 262, + 460, + 830, + 332, + 17, + 220, + 653, + 9057, + 198 + ], + "decoded": "def f(x):\n return x**2 # square\n" + }, + { + "text": "if (a != b && c <= d) { return std::max(a, b); }", + "ids": [ + 331, + 318, + 64, + 931, + 292, + 976, + 272, + 2564, + 293, + 8, + 313, + 460, + 1407, + 476, + 2721, + 2784, + 11, + 292, + 1174, + 333 + ], + "decoded": "if (a != b && c <= d) { return std::max(a, b); }" + }, + { + "text": "Ünïcödé naïve café résumé", + "ids": [ + 50724, + 77, + 245787, + 2862, + 164497, + 91603, + 571, + 50203, + 238976 + ], + "decoded": "Ünïcödé naïve café résumé" + }, + { + "text": "日本語のテキストと漢字", + "ids": [ + 247359, + 15303, + 210342, + 18602, + 101306, + 96356 + ], + "decoded": "日本語のテキストと漢字" + }, + { + "text": "Привет, как дела?", + "ids": [ + 51844, + 26125, + 7921, + 11, + 150914, + 170220, + 30 + ], + "decoded": "Привет, как дела?" + }, + { + "text": "مرحبا بالعالم", + "ids": [ + 148739, + 28850, + 150027, + 182946, + 149650 + ], + "decoded": "مرحبا بالعالم" + }, + { + "text": "नमस्ते दुनिया", + "ids": [ + 58069, + 84237, + 150104, + 153348, + 184642, + 235886 + ], + "decoded": "नमस्ते दुनिया" + }, + { + "text": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "ids": [ + 9008, + 97, + 244, + 9008, + 248, + 222, + 40958, + 59720, + 235, + 9008, + 237, + 121, + 321, + 1799, + 54, + 41, + 59720, + 102, + 373, + 235, + 88995, + 119 + ], + "decoded": "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻" + }, + { + "text": "I'm we're they've he'd she'll it's", + "ids": [ + 40, + 2688, + 567, + 2224, + 781, + 2908, + 551, + 4035, + 1292, + 3172, + 424, + 579 + ], + "decoded": "I'm we're they've he'd she'll it's" + }, + { + "text": "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "ids": [ + 248045, + 846, + 198, + 3710, + 369, + 220, + 17, + 10, + 17, + 30, + 248046 + ], + "decoded": "<|im_start|>user\nWhat is 2+2?<|im_end|>" + }, + { + "text": "", + "ids": [], + "decoded": "" + }, + { + "text": " ", + "ids": [ + 220 + ], + "decoded": " " + }, + { + "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ids": [ + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 67089, + 27528 + ], + "decoded": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] +} \ No newline at end of file diff --git a/tests/hf_tokenizer_test.cpp b/tests/hf_tokenizer_test.cpp new file mode 100644 index 00000000..a0216146 --- /dev/null +++ b/tests/hf_tokenizer_test.cpp @@ -0,0 +1,57 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// hf_tokenizer_test [ ...] +// +// Every case in each golden file (made by tools/make_tokenizer_golden.py with +// the Python `tokenizers` of the same release as third_party/tokenizers) must +// encode to the same ids through the C ABI, and decode back to the same text. +#include "hf_tokenizer.h" + +#include + +#include +#include + +int main(int argc, char** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: hf_tokenizer_test ...\n"); + return 2; + } + const onebit::HfTokenizer tok(argv[1]); + int n = 0, failures = 0; + for (int a = 2; a < argc; a++) { + std::ifstream f(argv[a]); + const auto golden = nlohmann::json::parse(f); + const bool add_special = golden.value("add_special", false); + for (const auto& c : golden.at("cases")) { + const std::string text = c.at("text"); + const auto want = c.at("ids").get>(); + const auto got = tok.encode(text, add_special); + ++n; + if (got != want) { + ++failures; + std::printf("FAIL encode %s\n want %s\n got %s\n", nlohmann::json(text).dump().c_str(), + nlohmann::json(want).dump().c_str(), nlohmann::json(got).dump().c_str()); + } else if (c.contains("decoded") && tok.decode(got) != c.at("decoded").get()) { + ++failures; + std::printf("FAIL decode %s -> %s\n", nlohmann::json(text).dump().c_str(), + nlohmann::json(tok.decode(got)).dump().c_str()); + } + } + } + std::printf("%d/%d cases match (vocab %zu)\n", n - failures, n, tok.vocab_size()); + return failures ? 1 : 0; +} diff --git a/third_party/tokenizers b/third_party/tokenizers new file mode 160000 index 00000000..88a4498a --- /dev/null +++ b/third_party/tokenizers @@ -0,0 +1 @@ +Subproject commit 88a4498ad4ea1a9487b0a9b0ff881383fd5a06a3 diff --git a/tools/copyright.py b/tools/copyright.py index 9711a04c..425318fa 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -30,13 +30,13 @@ # Comment style by file name pattern. LINE_STYLES = [ - (("*.cpp", "*.h", "*.inc", "*.hpp", "*.c", "*.cc"), "//"), + (("*.cpp", "*.h", "*.inc", "*.hpp", "*.c", "*.cc", "*.rs"), "//"), (("*.py", "*.sh", "*.yml", "*.yaml", "*.cmake", "*.cmake.in", "CMakeLists.txt", ".gitmodules", ".gitignore", ".gitattributes", ".clang-format", "*.toml", "*.config", "*.fragment"), "#"), ] BLOCK_STYLES = [(("*.md",), (""))] -EXEMPT = ["third_party/*", "patches/*", "LICENSE", "NOTICE", "*.gguf", "*.json", "*.tsv", "tests/golden/*/tokens.txt"] +EXEMPT = ["third_party/*", "patches/*", "LICENSE", "NOTICE", "*.gguf", "*.json", "*.tsv", "tests/golden/*/tokens.txt", "*/Cargo.lock"] def _match(name, patterns): diff --git a/tools/make_tokenizer_golden.py b/tools/make_tokenizer_golden.py new file mode 100755 index 00000000..7a35688d --- /dev/null +++ b/tools/make_tokenizer_golden.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# make_tokenizer_golden.py [source label] +# +# Golden cases for tests/hf_tokenizer_test.cpp, made with the Python +# `tokenizers` of the same release as third_party/tokenizers (pip install +# tokenizers==). The corpus mixes the Qwen3 golden texts with +# multilingual text, code, numbers, whitespace runs and emoji. +import json, sys +import tokenizers + +CORPUS = [ + "The capital of France is", + "Hello, world! How's it going?", + " leading and inner spaces\tand\ttabs\n\nnewlines", + "Numbers: 3.14159, 1,000,000 and 2026-09-23T17:30:00Z", + "def f(x):\n return x**2 # square\n", + "if (a != b && c <= d) { return std::max(a, b); }", + "Ünïcödé naïve café résumé", + "日本語のテキストと漢字", + "Привет, как дела?", + "مرحبا بالعالم", + "नमस्ते दुनिया", + "🤖🚀 emoji 👍🏽 and ZWJ 👩‍💻", + "I'm we're they've he'd she'll it's", + "<|im_start|>user\nWhat is 2+2?<|im_end|>", + "", + " ", + "a" * 300, +] + +tok_path, out = sys.argv[1], sys.argv[2] +label = sys.argv[3] if len(sys.argv) > 3 else tok_path +tok = tokenizers.Tokenizer.from_file(tok_path) +cases = [] +for text in CORPUS: + ids = tok.encode(text, add_special_tokens=False).ids + cases.append({"text": text, "ids": ids, "decoded": tok.decode(ids, skip_special_tokens=False)}) +json.dump({"source": f"tokenizers {tokenizers.__version__}, {label}", "add_special": False, "cases": cases}, + open(out, "w"), ensure_ascii=False, indent=1) +print(f"{out}: {len(cases)} cases, tokenizers {tokenizers.__version__}")