bong-water-water-bong · Sep 23, 2026 · Sep 23, 2026 · Sep 23, 2026
diff --git a/.github/workflows/bump-tokenizers.yml b/.github/workflows/bump-tokenizers.yml
@@ -0,0 +1,100 @@
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Keep Hugging Face tokenizers current (docs/tokenizers.md): when
+# huggingface/tokenizers publishes a newer release, open a PR that moves
+# third_party/tokenizers to it and refreshes hf_tokenizers/Cargo.lock. Before
+# merging, regenerate the golden cases with the same Python release
+# (tools/make_tokenizer_golden.py) and run the hf_tokenizer tests.
+#
+# Uses the secret HRX_BUMP_TOKEN (Contents and Pull requests read/write on
+# 1bit-MONSTER/engine): a PR opened with the default GITHUB_TOKEN would not
+# start CI.
+name: bump-tokenizers
+
+on:
+  schedule:
+    - cron: "5 8 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: bump-tokenizers
+  cancel-in-progress: false
+
+jobs:
+  bump:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require the token
+        env:
+          HRX_BUMP_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }}
+        run: |
+          if [ -z "$HRX_BUMP_TOKEN" ]; then
+            echo "::error::secret HRX_BUMP_TOKEN is not set (see the header of this workflow)"
+            exit 1
+          fi
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.HRX_BUMP_TOKEN }}
+          submodules: false
+
+      - name: Compare with the latest release
+        id: pins
+        env:
+          GH_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }}
+        run: |
+          set -euo pipefail
+          tag=$(gh api repos/huggingface/tokenizers/releases/latest --jq .tag_name)
+          upstream=$(git ls-remote https://github.com/huggingface/tokenizers.git "refs/tags/${tag}^{}" | awk '{print $1}')
+          [ -n "$upstream" ] || upstream=$(git ls-remote https://github.com/huggingface/tokenizers.git "refs/tags/${tag}" | awk '{print $1}')
+          ours=$(git ls-tree HEAD third_party/tokenizers | awk '{print $3}')
+          echo "latest $tag = $upstream, ours $ours"
+          {
+            echo "tag=$tag"; echo "upstream=$upstream"; echo "ours=$ours"
+            if [ "$upstream" = "$ours" ]; then echo "changed=false"; else echo "changed=true"; fi
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Open the bump PR
+        if: steps.pins.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.HRX_BUMP_TOKEN }}
+          TAG: ${{ steps.pins.outputs.tag }}
+          UPSTREAM: ${{ steps.pins.outputs.upstream }}
+          OURS: ${{ steps.pins.outputs.ours }}
+        run: |
+          set -euo pipefail
+          branch="bump-tokenizers/${TAG}"
+          if git ls-remote --exit-code origin "refs/heads/$branch" > /dev/null; then
+            echo "$branch already exists"; exit 0
+          fi
+          git config user.name "tokenizers-bump"
+          git config user.email "tokenizers-bump@users.noreply.github.com"
+          git switch -c "$branch"
+          git submodule update --init --depth 1 third_party/tokenizers
+          git -C third_party/tokenizers fetch -q --depth 1 origin "$UPSTREAM"
+          git -C third_party/tokenizers checkout -q "$UPSTREAM"
+          (cd hf_tokenizers && cargo update -p tokenizers)
+          git add third_party/tokenizers hf_tokenizers/Cargo.lock
+          git commit -q -m "Bump tokenizers: huggingface/tokenizers ${TAG} (${UPSTREAM:0:12})"
+          git push -q origin "$branch"
+          gh pr create --base main --head "$branch" \
+            --title "Bump tokenizers: ${TAG}" \
+            --body "Moves third_party/tokenizers from \`${OURS:0:12}\` to \`${TAG}\` (\`${UPSTREAM:0:12}\`) and refreshes hf_tokenizers/Cargo.lock.
+
+          Before merging: regenerate tests/data/tokenizers/*.json with Python tokenizers of the same release (tools/make_tokenizer_golden.py) and run ctest -R hf_tokenizer with -DONEBIT_TOKENIZER_ROOT."
diff --git a/.gitmodules b/.gitmodules
@@ -48,4 +48,10 @@
 	path = third_party/laya
 	url = https://github.com/NandhaKishorM/laya.git
 	branch = main
+# Hugging Face tokenizers (docs/tokenizers.md): the Rust library behind the
+# tokenizers npm/PyPI packages, same release; built by hf_tokenizers/ through
+# CMake (-DONEBIT_HF_TOKENIZERS=ON); .github/workflows/bump-tokenizers.yml keeps it current.
+[submodule "third_party/tokenizers"]
+	path = third_party/tokenizers
+	url = https://github.com/huggingface/tokenizers.git
 	shallow = true
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -74,6 +74,31 @@ if(ONEBIT_ZINC)
              COMMAND ${CMAKE_SOURCE_DIR}/tests/zinc_lemonade_e2e.sh $<TARGET_FILE:onebit> ${ONEBIT_ZINC_SERVER})
 endif()
 
+# ── Hugging Face tokenizers (docs/tokenizers.md) ───────────────────────────
+# Off by default: it needs cargo. Any model's tokenizer.json, through our C ABI
+# over third_party/tokenizers (the same release as the tokenizers npm/PyPI
+# packages). tests/data/tokenizers/ holds golden cases for 18 models.
+option(ONEBIT_HF_TOKENIZERS "Build Hugging Face tokenizers (Rust) behind a C ABI" OFF)
+if(ONEBIT_HF_TOKENIZERS)
+    find_package(Threads REQUIRED)
+    include(cmake/hf_tokenizers.cmake)
+    add_executable(hf_tokenizer_test tests/hf_tokenizer_test.cpp)
+    target_link_libraries(hf_tokenizer_test PRIVATE onebit_hf_tokenizers nlohmann_json::nlohmann_json)
+    # Golden cases whose tokenizer.json is under a model root (ONEBIT_TOKENIZER_ROOT,
+    # e.g. ~/.config/flm/models: <root>/<model>/tokenizer.json).
+    set(ONEBIT_TOKENIZER_ROOT "" CACHE PATH "Directory of model dirs holding tokenizer.json, for hf_tokenizer tests")
+    if(ONEBIT_TOKENIZER_ROOT)
+        file(GLOB _tok_golden "${CMAKE_SOURCE_DIR}/tests/data/tokenizers/*.json")
+        foreach(_g ${_tok_golden})
+            get_filename_component(_m "${_g}" NAME_WLE)
+            if(EXISTS "${ONEBIT_TOKENIZER_ROOT}/${_m}/tokenizer.json")
+                add_test(NAME hf_tokenizer_${_m}
+                         COMMAND hf_tokenizer_test ${ONEBIT_TOKENIZER_ROOT}/${_m}/tokenizer.json ${_g})
+            endif()
+        endforeach()
+    endif()
+endif()
+
 # ── Step 3: NPU engine, full ELFs only (docs/npu.md) ──────────────────────
 # Host side (full-ELF generation, model reader, buffer packing): pure C++, no
 # XRT. It needs PCRE2 (the tokenizer), and macOS has no NPU, so it is off there

diff --git a/cmake/hf_tokenizers.cmake b/cmake/hf_tokenizers.cmake
@@ -0,0 +1,40 @@
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Hugging Face tokenizers (docs/tokenizers.md): builds hf_tokenizers/ (our C ABI
+# over third_party/tokenizers) with cargo, --locked, into the build tree, and
+# exposes it as the onebit_hf_tokenizers target. rustup honours
+# hf_tokenizers/rust-toolchain.toml, which pins the Rust release.
+find_program(CARGO cargo REQUIRED)
+if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/tokenizers/tokenizers/Cargo.toml")
+    message(FATAL_ERROR "ONEBIT_HF_TOKENIZERS needs third_party/tokenizers: git submodule update --init third_party/tokenizers")
+endif()
+set(_hf_target_dir "${CMAKE_BINARY_DIR}/hf_tokenizers")
+set(_hf_lib "${_hf_target_dir}/release/${CMAKE_STATIC_LIBRARY_PREFIX}onebit_hf_tokenizers${CMAKE_STATIC_LIBRARY_SUFFIX}")
+file(GLOB_RECURSE _hf_sources CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/hf_tokenizers/src/*.rs")
+add_custom_command(
+    OUTPUT "${_hf_lib}"
+    COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_hf_target_dir}
+            ${CARGO} build --release --locked --manifest-path ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.toml
+    DEPENDS ${_hf_sources} ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.toml ${CMAKE_SOURCE_DIR}/hf_tokenizers/Cargo.lock
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/hf_tokenizers
+    COMMENT "cargo build hf_tokenizers (Hugging Face tokenizers)"
+    USES_TERMINAL)
+add_custom_target(hf_tokenizers_build DEPENDS "${_hf_lib}")
+add_library(onebit_hf_tokenizers STATIC IMPORTED GLOBAL)
+set_target_properties(onebit_hf_tokenizers PROPERTIES IMPORTED_LOCATION "${_hf_lib}")
+target_include_directories(onebit_hf_tokenizers INTERFACE "${CMAKE_SOURCE_DIR}/hf_tokenizers/include")
+target_link_libraries(onebit_hf_tokenizers INTERFACE Threads::Threads ${CMAKE_DL_LIBS} m)
+add_dependencies(onebit_hf_tokenizers hf_tokenizers_build)
diff --git a/docs/tokenizers.md b/docs/tokenizers.md
@@ -0,0 +1,56 @@
+<!--
+Copyright 2026 bong-water-water-bong
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Hugging Face tokenizers
+
+The engine reads any model's `tokenizer.json` through Hugging Face
+`tokenizers`, the Rust library behind the `tokenizers` npm and PyPI packages,
+at the same release as those packages.
+
+- `third_party/tokenizers` = `huggingface/tokenizers` **v0.23.2** (`88a4498a`).
+- `hf_tokenizers/` is our C ABI over it: `onebit_tok_from_json`, `_encode`,
+  `_decode`, `_vocab_size` and `_last_error`. It is built as a static library
+  with `default-features = false, features = ["onig"]`; the progress bar and
+  the unigram-training accelerator are for training only.
+  `hf_tokenizers/include/hf_tokenizer.h` adds a small C++ owner,
+  `onebit::HfTokenizer`.
+- The Rust release is pinned in `hf_tokenizers/rust-toolchain.toml` (1.98.1;
+  rustup honours it) and the crate graph in `hf_tokenizers/Cargo.lock`
+  (`cargo build --locked`).
+- `bump-tokenizers.yml` moves the pin to each new release and refreshes the lock.
+
+There is no official C binding upstream. mlc-ai/tokenizers-cpp wraps the same
+crate, but it pins 0.21.2.
+
+## Build and test
+
+```sh
+cmake -B build -DONEBIT_HF_TOKENIZERS=ON -DONEBIT_TOKENIZER_ROOT=~/.config/flm/models
+cmake --build build --target hf_tokenizer_test && (cd build && ctest -R hf_tokenizer)
+```
+
+`tests/data/tokenizers/<model>.json` holds golden cases made with Python
+`tokenizers==0.23.2` by `tools/make_tokenizer_golden.py`. The corpus mixes
+English, code, numbers, whitespace runs, eight scripts, emoji with ZWJ and
+chat markup. Each file names the sha256 of the tokenizer.json it came from.
+
+## Verified 2026-09-23
+
+18 of 18 models, 17 of 17 cases each, ids and decoded text identical:
+Qwen3 0.6B/1.7B/4B/8B, Qwen3-VL-4B, Qwen3.5-4B, Qwen3.6-35B-A3B, Llama 3.1-8B
+and 3.2-1B/3B, Gemma3 1B/4B, Gemma4 E2B/E4B, Phi4-mini, LFM2 1.2B/2.6B and
+Nanbeige4.1-3B. That covers vocabularies from 64k to 262k. The engine's own
+Qwen golden (`tests/data/qwen3_tokenizer_golden.json`) also matches, 14 of 14.