|
| 1 | +#include "tokenizer.h" |
| 2 | + |
| 3 | +#include <algorithm> |
| 4 | +#include <climits> |
| 5 | +#include <format> |
| 6 | +#include <variant> |
| 7 | + |
| 8 | +#include "unicode.h" |
| 9 | + |
| 10 | +namespace onebit { |
| 11 | + |
| 12 | +namespace { |
| 13 | + |
| 14 | +// llama.cpp token types stored in tokenizer.ggml.token_type. |
| 15 | +constexpr int32_t kTypeControl = 3; |
| 16 | +constexpr int32_t kTypeUserDefined = 4; |
| 17 | + |
| 18 | +using unicode::is_letter; |
| 19 | +using unicode::is_number; |
| 20 | +using unicode::is_whitespace; |
| 21 | + |
| 22 | +bool is_newline(char32_t c) { return c == U'\r' || c == U'\n'; } |
| 23 | + |
| 24 | +// Simple case folding for the letters the contraction branch can meet. |
| 25 | +// U+017F LATIN SMALL LETTER LONG S folds to 's' under (?i). |
| 26 | +char32_t fold(char32_t c) { |
| 27 | + if (c >= U'A' && c <= U'Z') return c + 32; |
| 28 | + if (c == 0x017F) return U's'; |
| 29 | + return c; |
| 30 | +} |
| 31 | + |
| 32 | +// Length of the pre-token starting at s[i] for the Qwen2 split pattern |
| 33 | +// (?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N} |
| 34 | +// | ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+ |
| 35 | +// Each branch is tried in order, as the regex engine does, and each returns |
| 36 | +// the length its greedy quantifiers settle on after backtracking. |
| 37 | +size_t qwen2_match(std::u32string_view s, size_t i) { |
| 38 | + const size_t n = s.size(); |
| 39 | + auto at = [&](size_t k) -> char32_t { return k < n ? s[k] : 0; }; |
| 40 | + auto is_other = [](char32_t c) { return !is_whitespace(c) && !is_letter(c) && !is_number(c); }; |
| 41 | + |
| 42 | + // (?i:'s|'t|'re|'ve|'m|'ll|'d) |
| 43 | + if (s[i] == U'\'' && i + 1 < n) { |
| 44 | + const char32_t a = fold(at(i + 1)), b = fold(at(i + 2)); |
| 45 | + if (a == U's' || a == U't') return 2; |
| 46 | + if ((a == U'r' && b == U'e') || (a == U'v' && b == U'e')) return 3; |
| 47 | + if (a == U'm') return 2; |
| 48 | + if (a == U'l' && b == U'l') return 3; |
| 49 | + if (a == U'd') return 2; |
| 50 | + } |
| 51 | + // [^\r\n\p{L}\p{N}]?\p{L}+ |
| 52 | + { |
| 53 | + size_t j = i; |
| 54 | + const char32_t c = s[i]; |
| 55 | + if (!is_newline(c) && !is_letter(c) && !is_number(c) && is_letter(at(i + 1))) j = i + 1; |
| 56 | + if (j < n && is_letter(s[j])) { |
| 57 | + while (j < n && is_letter(s[j])) ++j; |
| 58 | + return j - i; |
| 59 | + } |
| 60 | + } |
| 61 | + // \p{N} |
| 62 | + if (is_number(s[i])) return 1; |
| 63 | + // ' ?[^\s\p{L}\p{N}]+[\r\n]*' |
| 64 | + { |
| 65 | + size_t j = i; |
| 66 | + if (s[i] == U' ' && i + 1 < n && is_other(s[i + 1])) j = i + 1; |
| 67 | + if (j < n && is_other(s[j])) { |
| 68 | + while (j < n && is_other(s[j])) ++j; |
| 69 | + while (j < n && is_newline(s[j])) ++j; |
| 70 | + return j - i; |
| 71 | + } |
| 72 | + } |
| 73 | + // The remaining branches all start with whitespace. |
| 74 | + size_t ws_end = i; |
| 75 | + while (ws_end < n && is_whitespace(s[ws_end])) ++ws_end; |
| 76 | + if (ws_end == i) return 1; // unreachable: every code point matches a branch above |
| 77 | + // \s*[\r\n]+ : ends just after the last newline in the whitespace run. |
| 78 | + for (size_t k = ws_end; k > i; --k) |
| 79 | + if (is_newline(s[k - 1])) return k - i; |
| 80 | + // \s+(?!\S) |
| 81 | + if (ws_end == n) return ws_end - i; |
| 82 | + if (ws_end - i >= 2) return ws_end - i - 1; |
| 83 | + // \s+ |
| 84 | + return ws_end - i; |
| 85 | +} |
| 86 | + |
| 87 | +// GPT-2 bytes_to_unicode: printable bytes map to themselves, the rest to 256+. |
| 88 | +std::vector<char32_t> byte_to_codepoint() { |
| 89 | + std::vector<char32_t> map(256, 0); |
| 90 | + std::vector<bool> direct(256, false); |
| 91 | + for (int b = '!'; b <= '~'; ++b) direct[b] = true; |
| 92 | + for (int b = 0xA1; b <= 0xAC; ++b) direct[b] = true; |
| 93 | + for (int b = 0xAE; b <= 0xFF; ++b) direct[b] = true; |
| 94 | + char32_t next = 256; |
| 95 | + for (int b = 0; b < 256; ++b) map[b] = direct[b] ? char32_t(b) : next++; |
| 96 | + return map; |
| 97 | +} |
| 98 | + |
| 99 | +} // namespace |
| 100 | + |
| 101 | +std::expected<Tokenizer, std::string> Tokenizer::from_gguf(const GgufFile& f) { |
| 102 | + auto model = f.get_string("tokenizer.ggml.model"); |
| 103 | + if (model != "gpt2") return std::unexpected(std::format("tokenizer model '{}' is not supported", model.value_or(""))); |
| 104 | + auto pre = f.get_string("tokenizer.ggml.pre"); |
| 105 | + if (pre != "qwen2") return std::unexpected(std::format("pre-tokenizer '{}' is not supported", pre.value_or(""))); |
| 106 | + |
| 107 | + const GgufArray* tokens = f.get_array("tokenizer.ggml.tokens"); |
| 108 | + const GgufArray* types = f.get_array("tokenizer.ggml.token_type"); |
| 109 | + const GgufArray* merges = f.get_array("tokenizer.ggml.merges"); |
| 110 | + if (!tokens || !types || !merges || types->size() != tokens->size()) |
| 111 | + return std::unexpected(std::string("GGUF tokenizer arrays missing or inconsistent")); |
| 112 | + |
| 113 | + Tokenizer t; |
| 114 | + t.tokens_.reserve(tokens->size()); |
| 115 | + t.token_type_.reserve(tokens->size()); |
| 116 | + for (size_t i = 0; i < tokens->size(); ++i) { |
| 117 | + const auto* s = std::get_if<std::string>(&(*tokens)[i].v); |
| 118 | + const auto* ty = std::get_if<int64_t>(&(*types)[i].v); |
| 119 | + if (!s || !ty) return std::unexpected(std::format("bad tokenizer entry {}", i)); |
| 120 | + t.tokens_.push_back(*s); |
| 121 | + t.token_type_.push_back(int32_t(*ty)); |
| 122 | + t.token_id_.emplace(*s, int32_t(i)); |
| 123 | + if (*ty == kTypeControl || *ty == kTypeUserDefined) { |
| 124 | + auto cps = unicode::decode_utf8(*s); |
| 125 | + if (!cps || cps->empty()) return std::unexpected(std::format("added token {} is not valid UTF-8", i)); |
| 126 | + t.added_.emplace_back(std::u32string(cps->begin(), cps->end()), int32_t(i)); |
| 127 | + } |
| 128 | + } |
| 129 | + std::stable_sort(t.added_.begin(), t.added_.end(), |
| 130 | + [](const auto& a, const auto& b) { return a.first.size() > b.first.size(); }); |
| 131 | + |
| 132 | + t.merge_rank_.reserve(merges->size()); |
| 133 | + for (size_t i = 0; i < merges->size(); ++i) { |
| 134 | + const auto* m = std::get_if<std::string>(&(*merges)[i].v); |
| 135 | + if (!m || std::count(m->begin(), m->end(), ' ') != 1) return std::unexpected(std::format("bad merge {}", i)); |
| 136 | + t.merge_rank_.emplace(*m, int32_t(i)); |
| 137 | + } |
| 138 | + |
| 139 | + const auto map = byte_to_codepoint(); |
| 140 | + for (int b = 0; b < 256; ++b) { |
| 141 | + unicode::append_utf8(t.byte_to_unicode_[b], map[b]); |
| 142 | + t.unicode_to_byte_[map[b]] = uint8_t(b); |
| 143 | + } |
| 144 | + for (int b = 0; b < 256; ++b) |
| 145 | + if (!t.token_id_.contains(t.byte_to_unicode_[b])) |
| 146 | + return std::unexpected(std::format("vocab has no token for byte {}", b)); |
| 147 | + |
| 148 | + auto id_or = [&](const char* key) { |
| 149 | + auto v = f.get_uint(key); |
| 150 | + return (v && *v < t.tokens_.size()) ? int32_t(*v) : -1; |
| 151 | + }; |
| 152 | + t.bos_ = id_or("tokenizer.ggml.bos_token_id"); |
| 153 | + t.eos_ = id_or("tokenizer.ggml.eos_token_id"); |
| 154 | + return t; |
| 155 | +} |
| 156 | + |
| 157 | +void Tokenizer::bpe(const std::string& piece, std::vector<int32_t>& out) const { |
| 158 | + // Split into the mapped characters (each 1-2 UTF-8 bytes long). |
| 159 | + std::vector<std::string> sym; |
| 160 | + for (size_t i = 0; i < piece.size();) { |
| 161 | + const size_t len = (static_cast<unsigned char>(piece[i]) < 0x80) ? 1 : 2; |
| 162 | + sym.push_back(piece.substr(i, len)); |
| 163 | + i += len; |
| 164 | + } |
| 165 | + // Repeatedly merge the adjacent pair with the lowest rank (leftmost on ties). |
| 166 | + std::string key; |
| 167 | + while (sym.size() > 1) { |
| 168 | + int32_t best = INT32_MAX; |
| 169 | + size_t at = 0; |
| 170 | + for (size_t i = 0; i + 1 < sym.size(); ++i) { |
| 171 | + key.assign(sym[i]).append(" ").append(sym[i + 1]); |
| 172 | + auto it = merge_rank_.find(key); |
| 173 | + if (it != merge_rank_.end() && it->second < best) { |
| 174 | + best = it->second; |
| 175 | + at = i; |
| 176 | + } |
| 177 | + } |
| 178 | + if (best == INT32_MAX) break; |
| 179 | + sym[at] += sym[at + 1]; |
| 180 | + sym.erase(sym.begin() + std::ptrdiff_t(at) + 1); |
| 181 | + } |
| 182 | + for (const auto& s : sym) out.push_back(token_id_.at(s)); // single bytes are checked at load |
| 183 | +} |
| 184 | + |
| 185 | +void Tokenizer::split(std::u32string_view text, std::vector<std::string>& pieces) const { |
| 186 | + const std::vector<char32_t> norm = unicode::nfc(std::vector<char32_t>(text.begin(), text.end())); |
| 187 | + const std::u32string_view s(norm.data(), norm.size()); |
| 188 | + std::string utf8; |
| 189 | + for (size_t i = 0; i < s.size();) { |
| 190 | + const size_t len = qwen2_match(s, i); |
| 191 | + utf8.clear(); |
| 192 | + for (size_t k = i; k < i + len; ++k) unicode::append_utf8(utf8, s[k]); |
| 193 | + std::string& piece = pieces.emplace_back(); |
| 194 | + for (unsigned char b : utf8) piece += byte_to_unicode_[b]; |
| 195 | + i += len; |
| 196 | + } |
| 197 | +} |
| 198 | + |
| 199 | +void Tokenizer::encode_plain(std::u32string_view text, std::vector<int32_t>& out) const { |
| 200 | + std::vector<std::string> pieces; |
| 201 | + split(text, pieces); |
| 202 | + for (const auto& piece : pieces) bpe(piece, out); |
| 203 | +} |
| 204 | + |
| 205 | +std::expected<std::vector<std::string>, std::string> Tokenizer::pre_tokenize(std::string_view text) const { |
| 206 | + auto cps = unicode::decode_utf8(text); |
| 207 | + if (!cps) return std::unexpected(std::string("input is not valid UTF-8")); |
| 208 | + std::vector<std::string> pieces; |
| 209 | + split(std::u32string_view(cps->data(), cps->size()), pieces); |
| 210 | + return pieces; |
| 211 | +} |
| 212 | + |
| 213 | +std::expected<std::vector<int32_t>, std::string> Tokenizer::encode(std::string_view text, bool parse_special) const { |
| 214 | + auto cps = unicode::decode_utf8(text); |
| 215 | + if (!cps) return std::unexpected(std::string("input is not valid UTF-8")); |
| 216 | + const std::u32string_view s(cps->data(), cps->size()); |
| 217 | + |
| 218 | + std::vector<int32_t> out; |
| 219 | + size_t start = 0; |
| 220 | + if (parse_special) { |
| 221 | + // Leftmost-longest match of added tokens, as HF's AddedVocabulary does. |
| 222 | + for (size_t i = 0; i < s.size();) { |
| 223 | + const std::pair<std::u32string, int32_t>* hit = nullptr; |
| 224 | + for (const auto& a : added_) |
| 225 | + if (s.substr(i, a.first.size()) == a.first) { |
| 226 | + hit = &a; |
| 227 | + break; |
| 228 | + } |
| 229 | + if (!hit) { |
| 230 | + ++i; |
| 231 | + continue; |
| 232 | + } |
| 233 | + encode_plain(s.substr(start, i - start), out); |
| 234 | + out.push_back(hit->second); |
| 235 | + i += hit->first.size(); |
| 236 | + start = i; |
| 237 | + } |
| 238 | + } |
| 239 | + encode_plain(s.substr(start), out); |
| 240 | + return out; |
| 241 | +} |
| 242 | + |
| 243 | +std::expected<std::string, std::string> Tokenizer::decode(std::span<const int32_t> ids, bool skip_special) const { |
| 244 | + std::string out; |
| 245 | + for (int32_t id : ids) { |
| 246 | + if (id < 0 || size_t(id) >= tokens_.size()) return std::unexpected(std::format("token id {} out of range", id)); |
| 247 | + const int32_t type = token_type_[size_t(id)]; |
| 248 | + if (type == kTypeControl || type == kTypeUserDefined) { |
| 249 | + if (!(skip_special && type == kTypeControl)) out += tokens_[size_t(id)]; |
| 250 | + continue; |
| 251 | + } |
| 252 | + auto cps = unicode::decode_utf8(tokens_[size_t(id)]); |
| 253 | + if (!cps) return std::unexpected(std::format("token {} is not valid UTF-8", id)); |
| 254 | + for (char32_t cp : *cps) { |
| 255 | + auto it = unicode_to_byte_.find(cp); |
| 256 | + if (it == unicode_to_byte_.end()) return std::unexpected(std::format("token {} is not byte-level", id)); |
| 257 | + out += char(it->second); |
| 258 | + } |
| 259 | + } |
| 260 | + return out; |
| 261 | +} |
| 262 | + |
| 263 | +} // namespace onebit |
0 commit comments