diff --git a/maps/html4.json b/maps/html4.json
new file mode 100644
index 00000000..3a5e43ca
--- /dev/null
+++ b/maps/html4.json
@@ -0,0 +1,254 @@
+[
+ "Aacute",
+ "aacute",
+ "Acirc",
+ "acirc",
+ "acute",
+ "AElig",
+ "aelig",
+ "Agrave",
+ "agrave",
+ "alefsym",
+ "Alpha",
+ "alpha",
+ "amp",
+ "and",
+ "ang",
+ "Aring",
+ "aring",
+ "asymp",
+ "Atilde",
+ "atilde",
+ "Auml",
+ "auml",
+ "bdquo",
+ "Beta",
+ "beta",
+ "brvbar",
+ "bull",
+ "cap",
+ "Ccedil",
+ "ccedil",
+ "cedil",
+ "cent",
+ "Chi",
+ "chi",
+ "circ",
+ "clubs",
+ "cong",
+ "copy",
+ "crarr",
+ "cup",
+ "curren",
+ "dagger",
+ "Dagger",
+ "darr",
+ "dArr",
+ "deg",
+ "Delta",
+ "delta",
+ "diams",
+ "divide",
+ "Eacute",
+ "eacute",
+ "Ecirc",
+ "ecirc",
+ "Egrave",
+ "egrave",
+ "empty",
+ "emsp",
+ "ensp",
+ "Epsilon",
+ "epsilon",
+ "equiv",
+ "Eta",
+ "eta",
+ "ETH",
+ "eth",
+ "Euml",
+ "euml",
+ "euro",
+ "exist",
+ "fnof",
+ "forall",
+ "frac12",
+ "frac14",
+ "frac34",
+ "frasl",
+ "Gamma",
+ "gamma",
+ "ge",
+ "gt",
+ "harr",
+ "hArr",
+ "hearts",
+ "hellip",
+ "Iacute",
+ "iacute",
+ "Icirc",
+ "icirc",
+ "iexcl",
+ "Igrave",
+ "igrave",
+ "image",
+ "infin",
+ "int",
+ "Iota",
+ "iota",
+ "iquest",
+ "isin",
+ "Iuml",
+ "iuml",
+ "Kappa",
+ "kappa",
+ "Lambda",
+ "lambda",
+ "lang",
+ "laquo",
+ "larr",
+ "lArr",
+ "lceil",
+ "ldquo",
+ "le",
+ "lfloor",
+ "lowast",
+ "loz",
+ "lrm",
+ "lsaquo",
+ "lsquo",
+ "lt",
+ "macr",
+ "mdash",
+ "micro",
+ "middot",
+ "minus",
+ "Mu",
+ "mu",
+ "nabla",
+ "nbsp",
+ "ndash",
+ "ne",
+ "ni",
+ "not",
+ "notin",
+ "nsub",
+ "Ntilde",
+ "ntilde",
+ "Nu",
+ "nu",
+ "Oacute",
+ "oacute",
+ "Ocirc",
+ "ocirc",
+ "OElig",
+ "oelig",
+ "Ograve",
+ "ograve",
+ "oline",
+ "Omega",
+ "omega",
+ "Omicron",
+ "omicron",
+ "oplus",
+ "or",
+ "ordf",
+ "ordm",
+ "Oslash",
+ "oslash",
+ "Otilde",
+ "otilde",
+ "otimes",
+ "Ouml",
+ "ouml",
+ "para",
+ "part",
+ "permil",
+ "perp",
+ "Phi",
+ "phi",
+ "Pi",
+ "pi",
+ "piv",
+ "plusmn",
+ "pound",
+ "prime",
+ "Prime",
+ "prod",
+ "prop",
+ "Psi",
+ "psi",
+ "quot",
+ "radic",
+ "rang",
+ "raquo",
+ "rarr",
+ "rArr",
+ "rceil",
+ "rdquo",
+ "real",
+ "reg",
+ "rfloor",
+ "Rho",
+ "rho",
+ "rlm",
+ "rsaquo",
+ "rsquo",
+ "sbquo",
+ "Scaron",
+ "scaron",
+ "sdot",
+ "sect",
+ "shy",
+ "Sigma",
+ "sigma",
+ "sigmaf",
+ "sim",
+ "spades",
+ "sub",
+ "sube",
+ "sum",
+ "sup",
+ "sup1",
+ "sup2",
+ "sup3",
+ "supe",
+ "szlig",
+ "Tau",
+ "tau",
+ "there4",
+ "Theta",
+ "theta",
+ "thetasym",
+ "thinsp",
+ "THORN",
+ "thorn",
+ "tilde",
+ "times",
+ "trade",
+ "Uacute",
+ "uacute",
+ "uarr",
+ "uArr",
+ "Ucirc",
+ "ucirc",
+ "Ugrave",
+ "ugrave",
+ "uml",
+ "upsih",
+ "Upsilon",
+ "upsilon",
+ "Uuml",
+ "uuml",
+ "weierp",
+ "Xi",
+ "xi",
+ "Yacute",
+ "yacute",
+ "yen",
+ "yuml",
+ "Yuml",
+ "Zeta",
+ "zeta",
+ "zwj",
+ "zwnj"
+]
diff --git a/scripts/trie/decode-trie.ts b/scripts/trie/decode-trie.ts
index f5d99bd2..28c88357 100644
--- a/scripts/trie/decode-trie.ts
+++ b/scripts/trie/decode-trie.ts
@@ -90,29 +90,32 @@ export function decodeNode(
* or high (odd i) byte of slot i>>1.
*/
const packedKeySlots = Math.ceil(branchLength / 2);
+ const branchEnd = branchIndex + packedKeySlots + branchLength;
for (let keyIndex = 0; keyIndex < branchLength; keyIndex++) {
const slot = keyIndex >> 1;
const packed = decodeMap[branchIndex + slot];
const key = (packed >> ((keyIndex & 1) * 8)) & 0xff;
const destinationIndex = branchIndex + packedKeySlots + keyIndex;
+ // Pointers are relative to the end of the branch data.
decodeNode(
decodeMap,
resultMap,
prefix + String.fromCharCode(key),
- (destinationIndex + decodeMap[destinationIndex]) & 0xff_ff,
+ (branchEnd + decodeMap[destinationIndex]) & 0xff_ff,
);
}
} else {
+ const branchEnd = branchIndex + branchLength;
for (let index = 0; index < branchLength; index++) {
const stored = decodeMap[branchIndex + index];
if (stored !== 0) {
const code = jumpOffset + index;
- const p = branchIndex + index;
+ // Stored offsets are end-relative, +1 (0 = no branch).
decodeNode(
decodeMap,
resultMap,
prefix + String.fromCharCode(code),
- (p + stored - 1) & 0xff_ff,
+ (branchEnd + stored - 1) & 0xff_ff,
);
}
}
diff --git a/scripts/trie/encode-dict.spec.ts b/scripts/trie/encode-dict.spec.ts
new file mode 100644
index 00000000..53d62e0e
--- /dev/null
+++ b/scripts/trie/encode-dict.spec.ts
@@ -0,0 +1,104 @@
+import { describe, expect, it } from "vitest";
+import entityMap from "../../maps/entities.json" with { type: "json" };
+import legacyMap from "../../maps/legacy.json" with { type: "json" };
+import xmlMap from "../../maps/xml.json" with { type: "json" };
+import { decodeTrieDict } from "../../src/internal/decode-shared.js";
+import { encodeFullTrie, tryEncodeWithSplit } from "./encode-dict.js";
+import { encodeTrie } from "./encode-trie.js";
+import { getTrie } from "./trie.js";
+
+/*
+ * The dict/BPE string encoder here and the runtime `decodeTrieDict` in
+ * src/internal/decode-shared.ts are hand-synced implementations of the same
+ * format. These specs round-trip real and synthetic data through both, so a
+ * change to either side that breaks the agreement fails loudly instead of
+ * corrupting the shipped trie.
+ */
+
+/**
+ * Encode with the writer, decode with the runtime reader, and expect the
+ * original data back.
+ * @param data Uint16 values to round-trip.
+ * @param dictSize Optional fixed dictSize; defaults to the full search.
+ */
+function roundTrip(data: Uint16Array, dictSize?: number): void {
+ const result =
+ dictSize === undefined
+ ? encodeFullTrie(data)
+ : tryEncodeWithSplit(data, dictSize);
+ expect(result).not.toBeNull();
+ const decoded = decodeTrieDict(
+ result!.encoded,
+ data.length,
+ result!.atomCount,
+ result!.dict1AtomCount,
+ result!.ngramCount,
+ result!.dictSize,
+ );
+ expect(decoded).toStrictEqual(data);
+}
+
+describe("encode-dict ↔ decodeTrieDict round-trip", () => {
+ it("should round-trip the real HTML trie", () => {
+ const data = new Uint16Array(encodeTrie(getTrie(entityMap, legacyMap)));
+ roundTrip(data);
+ });
+
+ it("should round-trip the real XML trie", () => {
+ /*
+ * Shipped inline (too small for the dict), but the format must
+ * still round-trip; the tiny atom count needs a small dictSize.
+ */
+ const data = new Uint16Array(encodeTrie(getTrie(xmlMap, {})));
+ roundTrip(data, 10);
+ });
+
+ it("should round-trip repeated patterns (exercises BPE ngrams)", () => {
+ const values: number[] = [];
+ for (let index = 0; index < 200; index++) {
+ values.push(7, 1000, 42, 7, 1000, index % 5);
+ }
+ roundTrip(new Uint16Array(values), 4);
+ });
+
+ it("should round-trip large deltas (escape and double-escape)", () => {
+ /*
+ * Deltas: 88 (max 1-char), 89 (escape min), 8278 (escape max),
+ * 8279 (double-escape min), and a jump to the uint16 max.
+ */
+ const values = [0, 88, 177, 8455, 16_734, 65_535];
+ roundTrip(new Uint16Array([...values, ...values, ...values]), 3);
+ });
+
+ it("should round-trip long consecutive runs (RLE chunking)", () => {
+ /*
+ * A run of 200 consecutive values spans multiple RLE chunks
+ * (chunk cap is BASE + 1 = 92).
+ */
+ const values: number[] = [];
+ for (let index = 0; index < 200; index++) values.push(index);
+ for (let index = 0; index < 200; index++) values.push(index);
+ roundTrip(new Uint16Array(values), 40);
+ });
+
+ it("should round-trip across the whole dictSize search grid", () => {
+ const values: number[] = [];
+ for (let index = 0; index < 500; index++) {
+ values.push((index * 37) % 120, index % 60);
+ }
+ const data = new Uint16Array(values);
+ for (let dictSize = 45; dictSize <= 75; dictSize++) {
+ const result = tryEncodeWithSplit(data, dictSize);
+ if (result === null) continue;
+ const decoded = decodeTrieDict(
+ result.encoded,
+ data.length,
+ result.atomCount,
+ result.dict1AtomCount,
+ result.ngramCount,
+ result.dictSize,
+ );
+ expect(decoded).toStrictEqual(data);
+ }
+ });
+});
diff --git a/scripts/trie/encode-dict.ts b/scripts/trie/encode-dict.ts
new file mode 100644
index 00000000..50ac6e86
--- /dev/null
+++ b/scripts/trie/encode-dict.ts
@@ -0,0 +1,531 @@
+/*
+ * --- Encoded format -------------------------------------------------------
+ *
+ * The trie data (a Uint16Array) is encoded as a JS string of printable ASCII
+ * (91 chars: 0x21..0x7E minus `"`, `$`, `\`). Tokens are organised into:
+ *
+ * atoms — distinct uint16 values appearing in the data
+ * ngrams — BPE merges of two prior tokens (atoms or earlier ngrams)
+ *
+ * Each token sits in a slot indexed by code length:
+ *
+ * slots [0, A) dict1 atoms (1-char codes, top-`A` by use)
+ * slots [A, dictSize) dict1 ngrams (1-char codes, promoted ngrams)
+ * slots [dictSize, end) dict2 atoms then dict2 ngrams (both 2-char codes)
+ *
+ * Stream layout in the encoded string:
+ *
+ * [dict1 atoms: delta+RLE]
+ * [dict2 atoms: delta+RLE]
+ * [dict2 ngrams: each = 2 prior code refs]
+ * [dict1 ngrams: each = 2 prior code refs]
+ * [data: stream of slot codes]
+ *
+ * Decoding both atom dicts before either ngram dict, and decoding dict2
+ * ngrams before dict1 ngrams, lets every ngram entry reference any earlier
+ * slot without a forward reference. See `src/internal/decode-shared.ts` for
+ * the runtime decoder, which must stay in sync with this encoder
+ * (`encode-dict.spec.ts` round-trips the two).
+ *
+ * Compression comes from three layered effects:
+ * - Frequent atoms get 1-char slots (dict1).
+ * - BPE merges high-count pairs into ngrams that take fewer chars per use.
+ * - High-count merges may be _promoted_ into dict1 (1-char) by evicting the
+ * lowest-use dict1 atom; profitable when the per-use saving outweighs the
+ * evicted atom's per-use cost increase.
+ *
+ * The encoder constrains `dictSize + twoBytes = BASE`, so every slot code is
+ * either 1 or 2 chars (no 3-char escape range). This keeps the runtime
+ * decoder small and is sufficient as long as the trie fits in those slots.
+ */
+
+/** Printable ASCII chars safe in JS string literals (0x21..0x7E minus `"`, `$`, `\`). */
+const SAFE: number[] = [];
+for (let codePoint = 0x21; codePoint <= 0x7e; codePoint++) {
+ if (codePoint !== 0x22 && codePoint !== 0x24 && codePoint !== 0x5c) {
+ SAFE.push(codePoint);
+ }
+}
+const BASE = SAFE.length; // 91
+
+const RLE_MARKER = SAFE[89];
+const ESCAPE = SAFE[90];
+
+type Pair = readonly [number, number];
+
+// --- Base-91 slot codes ---------------------------------------------------
+
+/**
+ * Length in chars of a code referring to slot `s`. With our constraint
+ * `dictSize + twoBytes = BASE`, every slot code is 1 or 2 chars.
+ * @param slot
+ * @param dictSize
+ */
+function slotCodeLength(slot: number, dictSize: number): 1 | 2 {
+ return slot < dictSize ? 1 : 2;
+}
+
+/**
+ * Emit a code for slot `s` as a 1- or 2-char base-91 string.
+ * @param slot
+ * @param dictSize
+ */
+function emitSlotCode(slot: number, dictSize: number): string {
+ if (slot < dictSize) return String.fromCharCode(SAFE[slot]);
+ const r = slot - dictSize;
+ return String.fromCharCode(
+ SAFE[dictSize + Math.floor(r / BASE)],
+ SAFE[r % BASE],
+ );
+}
+
+// --- Delta + RLE encoding for the atom dict streams -----------------------
+
+/**
+ * Delta-encode a strictly-ascending list of integers as base-91 chars, with
+ * run-length compression for consecutive +1 deltas.
+ *
+ * delta < 89 → 1 char SAFE[delta]
+ * run of `n` ones (n ≥ 3) → 2 chars/chunk SAFE[89] SAFE[n-2]
+ * delta in [89, 8278] → 3 chars SAFE[90] SAFE[a] SAFE[b]
+ * larger delta → 5 chars SAFE[90] SAFE[90] SAFE[a] SAFE[b] SAFE[c]
+ * @param values
+ */
+function deltaRleEncode(values: number[]): string {
+ let out = "";
+ let previous = 0;
+ let index = 0;
+ while (index < values.length) {
+ const delta = values[index] - previous;
+ // RLE for runs of three or more consecutive +1 deltas.
+ if (delta === 1) {
+ let runLength = 1;
+ while (
+ index + runLength < values.length &&
+ values[index + runLength] - values[index + runLength - 1] === 1
+ ) {
+ runLength++;
+ }
+ if (runLength >= 3) {
+ let remaining = runLength;
+ while (remaining >= 3) {
+ const chunk = Math.min(remaining, BASE + 1);
+ out += String.fromCharCode(RLE_MARKER, SAFE[chunk - 2]);
+ remaining -= chunk;
+ }
+ for (let r = 0; r < remaining; r++) {
+ out += String.fromCharCode(SAFE[1]);
+ }
+ previous = values[index + runLength - 1];
+ index += runLength;
+ continue;
+ }
+ }
+ if (delta < 89) {
+ out += String.fromCharCode(SAFE[delta]);
+ } else {
+ const adjusted = delta - 89;
+ out +=
+ adjusted < 90 * BASE
+ ? String.fromCharCode(
+ ESCAPE,
+ SAFE[Math.floor(adjusted / BASE)],
+ SAFE[adjusted % BASE],
+ )
+ : String.fromCharCode(
+ ESCAPE,
+ ESCAPE,
+ SAFE[Math.floor(adjusted / (BASE * BASE))],
+ SAFE[Math.floor(adjusted / BASE) % BASE],
+ SAFE[adjusted % BASE],
+ );
+ }
+ previous = values[index];
+ index++;
+ }
+ return out;
+}
+
+// --- BPE: merge token pairs into ngrams -----------------------------------
+
+/** Encodes a pair (a, b) into a single number for use as Map keys. */
+const PAIR_RADIX = 1_000_003;
+
+/**
+ * Count pair occurrences in `seq`. For pairs (a, b) with a !== b, overlapping
+ * and non-overlapping counts coincide. For self-pairs (X, X), greedy
+ * left-to-right replacement only takes floor(L/2) per run of length L, so we
+ * count those properly per-run.
+ * @param seq
+ */
+function countPairs(seq: number[]): Map {
+ const counts = new Map();
+ for (let index = 0; index < seq.length - 1; index++) {
+ const a = seq[index];
+ const b = seq[index + 1];
+ if (a === b) continue;
+ const key = a * PAIR_RADIX + b;
+ counts.set(key, (counts.get(key) ?? 0) + 1);
+ }
+ for (let index = 0; index < seq.length; ) {
+ let end = index + 1;
+ while (end < seq.length && seq[end] === seq[index]) end++;
+ const length = end - index;
+ if (length >= 2) {
+ const key = seq[index] * PAIR_RADIX + seq[index];
+ counts.set(key, (counts.get(key) ?? 0) + (length >> 1));
+ }
+ index = end;
+ }
+ return counts;
+}
+
+/**
+ * Replace every non-overlapping occurrence of pair (a, b) in `seq` with the
+ * single token `replacement`, returning a fresh array.
+ * @param seq
+ * @param a
+ * @param b
+ * @param replacement
+ */
+function replacePair(
+ seq: number[],
+ a: number,
+ b: number,
+ replacement: number,
+): number[] {
+ const out: number[] = [];
+ for (let index = 0; index < seq.length; index++) {
+ if (
+ index + 1 < seq.length &&
+ seq[index] === a &&
+ seq[index + 1] === b
+ ) {
+ out.push(replacement);
+ index++;
+ } else {
+ out.push(seq[index]);
+ }
+ }
+ return out;
+}
+
+interface BpeResult {
+ /** Token sequence after all merges have been applied. */
+ seq: number[];
+ /** Each ngram, in BPE-add order. ngram `k` has token id `atomCount + k`. */
+ ngrams: Pair[];
+ /** Token IDs of the ngrams that were promoted into 1-char dict1 codes. */
+ promotedNgrams: Set;
+}
+
+/**
+ * Greedy BPE: iteratively merge the highest-saving pair until no merge yields
+ * positive net savings.
+ *
+ * For each candidate pair (a, b) with `c` occurrences and code lengths
+ * (`la`, `lb`), the loop considers two placements for the new ngram:
+ *
+ * dict2 (2-char code, no demotion): net = (la + lb - 2) * c - (la + lb)
+ * dict1 (1-char code, demote 1 atom): net = (la + lb - 1) * c - (la + lb) - demotedFreq
+ *
+ * The dict2 placement is only legal when neither component is already a
+ * dict1 ngram (forward references are forbidden — see decoder layout). The
+ * pair + placement with highest positive net wins each iteration. After every
+ * merge we recompute frequencies, code lengths, and `demotedFreq`.
+ * @param initialSeq
+ * @param atomCount
+ * @param dictSize
+ */
+function bpeOptimize(
+ initialSeq: number[],
+ atomCount: number,
+ dictSize: number,
+): BpeResult {
+ let seq = [...initialSeq];
+ const ngrams: Pair[] = [];
+ const promotedNgrams = new Set();
+
+ /** CodeLength[id] gets recomputed after every merge. */
+ let codeLength = new Int8Array(0);
+ let demotedFreq = 0;
+
+ function refreshPartition() {
+ const totalTokens = atomCount + ngrams.length;
+ const dict1AtomCount = dictSize - promotedNgrams.size;
+
+ /*
+ * Atom use = standalone occurrences in `seq` plus uses as a component
+ * inside ngram entries — every use pays its slot's code length.
+ */
+ const use = new Int32Array(atomCount);
+ for (const t of seq) if (t < atomCount) use[t]++;
+ for (const [a, b] of ngrams) {
+ if (a < atomCount) use[a]++;
+ if (b < atomCount) use[b]++;
+ }
+
+ const atomsByUse = Array.from(
+ { length: atomCount },
+ (_, index) => index,
+ // eslint-disable-next-line unicorn/no-array-sort -- TS lib doesn't expose toSorted yet
+ ).sort((x, y) => use[y] - use[x]);
+
+ codeLength = new Int8Array(totalTokens);
+ for (const [rank, id] of atomsByUse.entries()) {
+ const slot =
+ rank < dict1AtomCount
+ ? rank
+ : dictSize + (rank - dict1AtomCount);
+ codeLength[id] = slotCodeLength(slot, dictSize);
+ }
+ /*
+ * Each ngram's slot: 1-char if promoted, otherwise 2-char (or 3-char
+ * if dict2 overflows). Within both kinds, slots increase in BPE order.
+ */
+ let nextDict1Slot = dict1AtomCount;
+ let nextDict2Slot = dictSize + (atomCount - dict1AtomCount);
+ for (const k of ngrams.keys()) {
+ const id = atomCount + k;
+ const slot = promotedNgrams.has(id)
+ ? nextDict1Slot++
+ : nextDict2Slot++;
+ codeLength[id] = slotCodeLength(slot, dictSize);
+ }
+
+ // The atom that the next promotion would push out of dict1.
+ demotedFreq =
+ dict1AtomCount > 0 ? use[atomsByUse[dict1AtomCount - 1]] : 0;
+ }
+ refreshPartition();
+
+ /*
+ * Stop after `BPE_MERGE_CAP` merges, even if more would shrink raw bytes.
+ * Empirically, shipping all profitable merges (~256 of them) shrinks the
+ * minified bundle further but flattens the body's char-frequency
+ * distribution enough that gzip and brotli on the bundled output _grow_
+ * by 200–350 bytes vs. the baseline. Capping at ~25 keeps minified well
+ * under baseline while letting gzip stay below and brotli within ~30
+ * bytes — a much better wire-size trade for consumers that compress.
+ */
+ const BPE_MERGE_CAP = 25;
+ /*
+ * Brotli-aware overrides found by coordinate descent over the merge
+ * sequence. At each listed step we pick the Nth-best candidate by net
+ * raw savings instead of the greedy best, because the resulting merge
+ * sequence yields better full-bundle brotli once the decoder + trie
+ * are bundled. Re-tuned for the end-relative pointer data: worth -33
+ * brotli bytes vs pure greedy at equal gzip.
+ *
+ * These are only valid for the exact trie contents they were tuned on;
+ * `write-decode-map.ts` guards the encoded length so this fires loudly
+ * when the trie changes.
+ */
+ const BPE_RANK_OVERRIDES: Record = { 3: 2, 4: 5 };
+ interface Candidate {
+ net: number;
+ a: number;
+ b: number;
+ promote: boolean;
+ }
+ for (let mergeCount = 0; mergeCount < BPE_MERGE_CAP; mergeCount++) {
+ const counts = countPairs(seq);
+ const dict2NgramSlot = atomCount + ngrams.length;
+ const dict2Length = slotCodeLength(dict2NgramSlot, dictSize);
+ const canPromote = dictSize > promotedNgrams.size;
+
+ const candidates: Candidate[] = [];
+ for (const [key, count] of counts) {
+ // eslint-disable-next-line unicorn/no-break-in-nested-loop
+ if (count < 2) continue;
+ const a = Math.floor(key / PAIR_RADIX);
+ const b = key % PAIR_RADIX;
+ const sum = codeLength[a] + codeLength[b];
+ const isDict2Allowed = !(
+ promotedNgrams.has(a) || promotedNgrams.has(b)
+ );
+
+ const dict2Net = isDict2Allowed
+ ? (sum - dict2Length) * count - sum
+ : // eslint-disable-next-line unicorn/prefer-global-number-constants -- biome's useNumberNamespace enforces `Number.NEGATIVE_INFINITY`
+ Number.NEGATIVE_INFINITY;
+ const dict1Net = canPromote
+ ? (sum - 1) * count - sum - demotedFreq
+ : // eslint-disable-next-line unicorn/prefer-global-number-constants -- biome's useNumberNamespace enforces `Number.NEGATIVE_INFINITY`
+ Number.NEGATIVE_INFINITY;
+ const net = Math.max(dict1Net, dict2Net);
+ if (net > 0) {
+ candidates.push({
+ net,
+ a,
+ b,
+ promote: dict1Net > dict2Net,
+ });
+ }
+ }
+ if (candidates.length === 0) break;
+ candidates.sort((x, y) => y.net - x.net);
+ const rank = BPE_RANK_OVERRIDES[mergeCount] ?? 0;
+ const best = candidates[Math.min(rank, candidates.length - 1)];
+
+ const newId = atomCount + ngrams.length;
+ ngrams.push([best.a, best.b]);
+ if (best.promote) promotedNgrams.add(newId);
+ seq = replacePair(seq, best.a, best.b, newId);
+ refreshPartition();
+ }
+
+ return { seq, ngrams, promotedNgrams };
+}
+
+// --- Encoding tries -------------------------------------------------------
+
+/**
+ *
+ */
+export interface EncodedTrie {
+ encoded: string;
+ /** Number of distinct uint16 values stored across dict1 + dict2. */
+ atomCount: number;
+ /** Number of dict1 entries that are atoms (rest are promoted ngrams). */
+ dict1AtomCount: number;
+ ngramCount: number;
+ dictSize: number;
+}
+
+/**
+ * Encode the trie with a given `dictSize`: the top `dictSize` slots are
+ * 1-char codes, and the remaining `BASE - dictSize` first-byte values are
+ * 2-char codes. Returns `null` if the trie doesn't fit in this slot space.
+ * @param data Uint16 trie words to encode.
+ * @param dictSize Number of 1-char code slots.
+ */
+export function tryEncodeWithSplit(
+ data: Uint16Array,
+ dictSize: number,
+): EncodedTrie | null {
+ if (dictSize < 1 || dictSize >= BASE) return null;
+ const capacity = dictSize + (BASE - dictSize) * BASE;
+
+ // Map each distinct uint16 value to a token id.
+ const valueToId = new Map();
+ const idToValue: number[] = [];
+ for (const v of data) {
+ if (valueToId.has(v)) {
+ continue;
+ }
+
+ valueToId.set(v, idToValue.length);
+ idToValue.push(v);
+ }
+ const atomCount = idToValue.length;
+ if (capacity < atomCount) return null;
+
+ const seq = Array.from(data, (v) => valueToId.get(v)!);
+ const bpe = bpeOptimize(seq, atomCount, dictSize);
+ if (atomCount + bpe.ngrams.length > capacity) return null;
+
+ /*
+ * Partition atoms into dict1 (top-use, 1-char) and dict2 (rest, 2-char or
+ * 3-char). Within each partition, sort by VALUE so the delta+RLE stream
+ * stays compact. Total atom use = standalone uses + ngram-component uses.
+ */
+ const totalUse = new Int32Array(atomCount);
+ for (const t of bpe.seq) if (t < atomCount) totalUse[t]++;
+ for (const [a, b] of bpe.ngrams) {
+ if (a < atomCount) totalUse[a]++;
+ if (b < atomCount) totalUse[b]++;
+ }
+ interface AtomEntry {
+ id: number;
+ value: number;
+ use: number;
+ }
+ const atomEntries: AtomEntry[] = idToValue.map((value, id) => ({
+ id,
+ value,
+ use: totalUse[id],
+ }));
+ atomEntries.sort((x, y) => y.use - x.use || x.value - y.value);
+
+ const dict1AtomCount = dictSize - bpe.promotedNgrams.size;
+ /*
+ * `dict1 = atomEntries.slice(0, dict1AtomCount)` clamps to atomCount, but
+ * the header still reports dict1AtomCount. If dict1AtomCount exceeded the
+ * atoms actually emitted, the decoder's `decodeDelta(dict1AtomCount, 0)`
+ * would over-read into the following streams and corrupt the trie.
+ */
+ if (dict1AtomCount > atomCount) {
+ throw new Error(
+ `dict1AtomCount (${dict1AtomCount}) exceeds the atom count ` +
+ `(${atomCount}); the dict1 atom stream would under-fill and ` +
+ "the decoder would over-read the following streams.",
+ );
+ }
+ const byValue = (x: AtomEntry, y: AtomEntry) => x.value - y.value;
+ // eslint-disable-next-line unicorn/no-array-sort -- TS lib doesn't expose toSorted yet
+ const dict1 = atomEntries.slice(0, dict1AtomCount).sort(byValue);
+ // eslint-disable-next-line unicorn/no-array-sort -- TS lib doesn't expose toSorted yet
+ const dict2 = atomEntries.slice(dict1AtomCount).sort(byValue);
+
+ // Slot for every token: atoms by partition+value-rank, ngrams by promotion+BPE-order.
+ const slot = new Int32Array(atomCount + bpe.ngrams.length);
+ for (const [index, entry] of dict1.entries()) slot[entry.id] = index;
+ for (const [index, entry] of dict2.entries())
+ slot[entry.id] = dictSize + index;
+ let nextDict1NgramSlot = dict1AtomCount;
+ let nextDict2NgramSlot = dictSize + dict2.length;
+ for (let k = 0; k < bpe.ngrams.length; k++) {
+ const id = atomCount + k;
+ slot[id] = bpe.promotedNgrams.has(id)
+ ? nextDict1NgramSlot++
+ : nextDict2NgramSlot++;
+ }
+
+ const code = (id: number) => emitSlotCode(slot[id], dictSize);
+ const dict1AtomHeader = deltaRleEncode(dict1.map((entry) => entry.value));
+ const dict2AtomHeader = deltaRleEncode(dict2.map((entry) => entry.value));
+ let dict1NgramHeader = "";
+ let dict2NgramHeader = "";
+ for (let k = 0; k < bpe.ngrams.length; k++) {
+ const id = atomCount + k;
+ const [a, b] = bpe.ngrams[k];
+ const reference = code(a) + code(b);
+ if (bpe.promotedNgrams.has(id)) dict1NgramHeader += reference;
+ else dict2NgramHeader += reference;
+ }
+ let body = "";
+ for (const t of bpe.seq) body += code(t);
+
+ const encoded =
+ dict1AtomHeader +
+ dict2AtomHeader +
+ dict2NgramHeader +
+ dict1NgramHeader +
+ body;
+ return {
+ encoded,
+ atomCount,
+ dict1AtomCount,
+ ngramCount: bpe.ngrams.length,
+ dictSize,
+ };
+}
+
+/**
+ * Try a range of dictSize values and return the smallest encoding. The grid
+ * is narrow because the BPE inside `tryEncodeWithSplit` is the hot loop;
+ * empirically the optimum lives in this range for the HTML entity trie.
+ * @param data Uint16 trie words to encode.
+ */
+export function encodeFullTrie(data: Uint16Array): EncodedTrie {
+ let best: EncodedTrie | null = null;
+ for (let dictSize = 45; dictSize <= 75; dictSize++) {
+ const result = tryEncodeWithSplit(data, dictSize);
+ if (result && (!best || result.encoded.length < best.encoded.length)) {
+ best = result;
+ }
+ }
+ if (!best) throw new Error("No viable dictSize split found.");
+ return best;
+}
diff --git a/scripts/trie/encode-trie.spec.ts b/scripts/trie/encode-trie.spec.ts
index 26624770..c4cc75d2 100644
--- a/scripts/trie/encode-trie.spec.ts
+++ b/scripts/trie/encode-trie.spec.ts
@@ -2,8 +2,6 @@ import { describe, expect, it } from "vitest";
import { encodeTrie } from "./encode-trie.js";
import type { TrieNode } from "./trie.js";
-const NO_BRANCH_SENTINEL_RE = /no-branch sentinel/;
-
describe("encode_trie", () => {
it("should encode an empty node", () => {
expect(encodeTrie({})).toStrictEqual([0b0000_0000_0000_0000]);
@@ -56,12 +54,16 @@ describe("encode_trie", () => {
*
* [0] header: branchCount=2 → 2<<7 = 256
* [1] keys: 'A'(65) | ('b'(98)<<8)
- * [2] dest[0]: relative ptr → nodeA at index 4
- * [3] dest[1]: relative ptr → nodeC at index 5
+ * [2] dest[0]: end-relative ptr → nodeA at index 4
+ * [3] dest[1]: end-relative ptr → nodeC at index 5
* [4] nodeA: value "a" inline → 0x4000 | 97
- * [5] nodeC header: branchCount=1, dictionary (nodeA already encoded)
- * [6] key: 'c'(99) packed
- * [7] dest: relative ptr → nodeA at index 4 (wraps via uint16)
+ * [5] nodeC header: 1-slot jump table (single branch, but nodeA is
+ * already encoded so the inline single-branch form can't be used)
+ * → (1<<7) | jumpOffset 'c'(99)
+ * [6] slot: ptr → nodeA at index 4 (backwards, wraps via uint16)
+ *
+ * Dict pointers are relative to the end of the branch data
+ * (branchIndex + packedKeySlots + branchCount = 1 + 1 + 2 = 4).
*/
const result = encodeTrie(trie);
@@ -74,9 +76,9 @@ describe("encode_trie", () => {
expect((result[1] >> 8) & 0xff).toBe(98); // 'b'
// [4]: nodeA with inline value 'a'
expect(result[4]).toBe(0b0100_0000_0000_0000 | 97);
- // [2],[3]: relative pointers that resolve to valid node indices
- expect((2 + result[2]) & 0xff_ff).toBe(4); // Dest[0] → nodeA
- expect((3 + result[3]) & 0xff_ff).toBe(5); // Dest[1] → nodeC
+ // [2],[3]: end-relative pointers that resolve to valid node indices
+ expect((4 + result[2]) & 0xff_ff).toBe(4); // Dest[0] → nodeA
+ expect((4 + result[3]) & 0xff_ff).toBe(5); // Dest[1] → nodeC
});
it("should encode a disjoint recursive branch", () => {
@@ -89,9 +91,11 @@ describe("encode_trie", () => {
*
* [0] header: branchCount=2 → 2<<7 = 256
* [1] keys: '0'(48) | ('a'(97)<<8) = 48 + 24832 = 24880
- * [2] dest[0]: relative ptr back to self at 0 → (0−2+0x10000)%0x10000 = 65534
- * [3] dest[1]: relative ptr to {value:"a"} at 4 → (4−3) = 1
+ * [2] dest[0]: end-relative ptr back to self at 0 → (0−4+0x10000)%0x10000 = 65532
+ * [3] dest[1]: end-relative ptr to {value:"a"} at 4 → (4−4) = 0
* [4] node: value "a" (1-char, inline) → 0x4000 | 97 = 16481
+ *
+ * Branch data ends at index 4 (header + 1 key word + 2 pointers).
*/
const result = encodeTrie(recursiveTrie);
@@ -101,9 +105,9 @@ describe("encode_trie", () => {
expect(result[1] & 0xff).toBe(48);
expect((result[1] >> 8) & 0xff).toBe(97);
// Dest[0] points back to self (index 0) — wraps around via uint16
- expect((2 + result[2]) & 0xff_ff).toBe(0);
+ expect((4 + result[2]) & 0xff_ff).toBe(0);
// Dest[1] points to the leaf node
- expect((3 + result[3]) & 0xff_ff).toBe(4);
+ expect((4 + result[3]) & 0xff_ff).toBe(4);
// Leaf: inline value 'a'
expect(result[4]).toBe(0b0100_0000_0000_0000 | 97);
});
@@ -113,10 +117,8 @@ describe("encode_trie", () => {
* Chars 48('0'), 49('1'), 52('4'), 54('6'), 56('8'), 57('9').
* Range 48..57 = 10 slots for 6 entries → overhead 10/6 = 1.67 → jump table.
*
- * '0' points at a non-recursive leaf so the first slot's stored value
- * isn't 0. (A self-ref at pointerPos=1 with childOffset=0 collides
- * with the no-branch sentinel and the encoder asserts against it —
- * see the "self-ref that collides" test below.)
+ * Jump-table pointers are stored relative to the end of the branch
+ * array (index 11 here), + 1 so that 0 stays the no-branch sentinel.
*/
const leaf: TrieNode = { value: "a" };
const jumpRecursiveTrie: TrieNode = { next: new Map() };
@@ -140,9 +142,10 @@ describe("encode_trie", () => {
}
// '0' resolves to the leaf node carrying value "a".
+ const branchEnd = 11; // 1 header + 10 slots
const leafStored = slotFor(48);
expect(leafStored).not.toBe(0);
- const leafIndex = (1 + leafStored - 1) & 0xff_ff;
+ const leafIndex = (branchEnd + leafStored - 1) & 0xff_ff;
expect(result[leafIndex]).toBe(0b0100_0000_0000_0000 | 97);
// Self-ref slots all resolve back to the root node (index 0).
@@ -150,19 +153,73 @@ describe("encode_trie", () => {
const pointerPos = 1 + (char - 48);
const stored = result[pointerPos];
expect(stored).not.toBe(0); // Never the no-branch sentinel
- expect((pointerPos + stored - 1) & 0xff_ff).toBe(0);
+ expect((branchEnd + stored - 1) & 0xff_ff).toBe(0);
}
});
- it("should reject a jump-table self-ref that would collide with the no-branch sentinel", () => {
+ it("should encode adjacent jump-table self-refs without sentinel collision", () => {
/*
* Two adjacent self-refs → overhead 1, takes the jump-table path.
- * The first slot (pointerPos=1, childOffset=0) encodes to 0, which
- * collides with the "no branch" sentinel and must be caught.
+ * No stored pointer may equal 0, the "no branch" sentinel. With
+ * end-relative pointers that collision is structurally impossible
+ * (stored = 0 would require the child to start at the last slot of
+ * the branch array itself); this pins the hazard case, the first
+ * slot (pointerPos=1, childOffset=0).
*/
const recursive: TrieNode = { next: new Map() };
recursive.next!.set(48, recursive);
recursive.next!.set(49, recursive);
- expect(() => encodeTrie(recursive)).toThrow(NO_BRANCH_SENTINEL_RE);
+ const result = encodeTrie(recursive);
+ const branchEnd = 3; // 1 header + 2 slots
+ for (const pointerPos of [1, 2]) {
+ expect(result[pointerPos]).not.toBe(0);
+ expect((branchEnd + result[pointerPos] - 1) & 0xff_ff).toBe(0);
+ }
+ });
+
+ it("uses a jump table for a contiguous range of 63 children (the 6-bit BRANCH_LENGTH limit)", () => {
+ /*
+ * Span 63 equals branch count 63: fits both the jump-table length
+ * field and the overhead budget, so the node is a jump table whose
+ * header carries the first char in JUMP_TABLE and 63 in BRANCH_LENGTH.
+ */
+ const next = new Map();
+ for (let char = 1; char <= 63; char++) next.set(char, { value: "a" });
+ const header = encodeTrie({ next })[0];
+ // JUMP_TABLE field holds the first covered char.
+ expect(header & 0b0111_1111).toBe(1);
+ // BRANCH_LENGTH field holds the run length.
+ expect((header >> 7) & 0b0011_1111).toBe(63);
+ });
+
+ it("falls back to a dictionary when the jump-table span exceeds 63 but the branch count fits", () => {
+ /*
+ * 63 children spanning 64 chars (one gap at 63): the branch count
+ * still fits the 6-bit field, but the jump-table length (span) is 64,
+ * so the `jumpTableLength <= 63` guard forces the dictionary encoding
+ * (JUMP_TABLE field is 0).
+ */
+ const next = new Map();
+ for (let char = 1; char <= 62; char++) next.set(char, { value: "a" });
+ // Gap at 63; char 64 makes the span 64.
+ next.set(64, { value: "a" });
+ const header = encodeTrie({ next })[0];
+ // Dictionary node: JUMP_TABLE field is 0.
+ expect(header & 0b0111_1111).toBe(0);
+ // BRANCH_LENGTH field holds all 63 branches.
+ expect((header >> 7) & 0b0011_1111).toBe(63);
+ });
+
+ it("cannot encode a node with 64 children (exceeds the 6-bit BRANCH_LENGTH field)", () => {
+ /*
+ * 64 overflows the 6-bit branch-count field for both the jump-table
+ * and dictionary encodings, so encoding must fail loudly rather than
+ * silently corrupt the header.
+ */
+ const next = new Map();
+ for (let char = 1; char <= 64; char++) next.set(char, { value: "a" });
+ expect(() => encodeTrie({ next })).toThrow(
+ "Too many bits for branches",
+ );
});
});
diff --git a/scripts/trie/encode-trie.ts b/scripts/trie/encode-trie.ts
index e2acaec1..23be91d6 100644
--- a/scripts/trie/encode-trie.ts
+++ b/scripts/trie/encode-trie.ts
@@ -13,19 +13,28 @@ function binaryLength(integer: number): number {
/**
* Encode a trie into compact binary representation.
* @param trie Trie node map to encode.
- * @param maxJumpTableOverhead Maximum allowed jump-table overhead before using linear encoding.
+ * @param maxJumpTableOverhead Maximum allowed jump-table overhead before
+ * using linear encoding — either a constant or a per-node function (used
+ * to give hot nodes a more generous threshold than cold ones).
*/
-export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
+export function encodeTrie(
+ trie: TrieNode,
+ maxJumpTableOverhead: number | ((node: TrieNode) => number) = 2,
+): number[] {
const encodeCache = new Map();
const enc: number[] = [];
+ const overheadForNode =
+ typeof maxJumpTableOverhead === "function"
+ ? maxJumpTableOverhead
+ : () => maxJumpTableOverhead;
function encodeNode(node: TrieNode): number {
const cached = encodeCache.get(node);
if (cached != null) return cached;
const startIndex = enc.length;
encodeCache.set(node, startIndex);
- const nodeIndex = enc.length;
enc.push(0);
+ const nodeIndex = enc.length - 1;
if (node.value != null) {
let valueLength =
@@ -79,17 +88,20 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
) {
const runLength = runChars.length;
if (runLength > 63) {
- addBranches(node.next, nodeIndex);
+ addBranches(node, nodeIndex);
assert.strictEqual(nodeIndex, startIndex);
return startIndex;
}
const firstChar = runChars[0];
assert.ok(firstChar < 0x80, "run first char must be < 128");
- const maskedRunLength = runLength & 0x3f;
+ /*
+ * FLAG13 with VALUE_LENGTH=0 marks a compact run (the
+ * same bit means "semicolon required" on value nodes).
+ * runLength fits the 6-bit BRANCH_LENGTH field — the
+ * `> 63` case bailed out above.
+ */
enc[nodeIndex] =
- BinTrieFlags.FLAG13 | // Compact run flag (same bit position)
- (maskedRunLength << 7) |
- firstChar;
+ BinTrieFlags.FLAG13 | (runLength << 7) | firstChar;
for (let index = 1; index < runLength; index += 2) {
const low = runChars[index];
const high = runChars[index + 1];
@@ -100,15 +112,15 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
return startIndex;
}
}
- addBranches(node.next, nodeIndex);
+ addBranches(node, nodeIndex);
}
assert.strictEqual(nodeIndex, startIndex, "Has expected location");
return startIndex;
}
- function addBranches(next: Map, nodeIndex: number) {
- const branches = [...next];
+ function addBranches(node: TrieNode, nodeIndex: number) {
+ const branches = [...node.next!];
if (branches.length === 0) return;
branches.sort(([a], [b]) => a - b);
assert.ok(
@@ -127,25 +139,38 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
const jumpEndValue = branches[branches.length - 1][0];
const jumpTableLength = jumpEndValue - jumpOffset + 1;
const jumpTableOverhead = jumpTableLength / branches.length;
- if (jumpTableOverhead <= maxJumpTableOverhead) {
+ // BRANCH_LENGTH is 6 bits → jumpTableLength must fit in 63 too.
+ if (
+ jumpTableOverhead <= overheadForNode(node) &&
+ jumpTableLength <= 63
+ ) {
assert.ok(
- binaryLength(jumpOffset) <= 16,
- `Offset ${jumpOffset} too large at ${binaryLength(jumpOffset)}`,
+ binaryLength(jumpOffset) <= 7,
+ `Jump-table first char ${jumpOffset} needs ${binaryLength(
+ jumpOffset,
+ )} bits but the JUMP_TABLE field is only 7`,
);
enc[nodeIndex] |= (jumpTableLength << 7) | jumpOffset;
assert.ok(
- binaryLength(jumpTableLength) <= 7,
+ binaryLength(jumpTableLength) <= 6,
`Too many bits (${binaryLength(jumpTableLength)}) for branches`,
);
for (let index = 0; index < jumpTableLength; index++) enc.push(0);
const branchIndex = enc.length - jumpTableLength;
+ const branchEnd = branchIndex + jumpTableLength;
for (const [char, child] of branches) {
const relativeIndex = char - jumpOffset;
const pointerPos = branchIndex + relativeIndex;
const childOffset = encodeNode(child);
- // Store relative offset + 1 (0 = no branch sentinel).
+ /*
+ * Store the offset relative to the END of the branch array,
+ * + 1 (0 = no branch sentinel). End-relative beats
+ * slot-relative for compression: the common "child encoded
+ * immediately after the table" case becomes the constant 1
+ * regardless of which slot points at it.
+ */
const stored =
- (childOffset - pointerPos + 1 + 0x1_00_00) % 0x1_00_00;
+ (childOffset - branchEnd + 1 + 0x1_00_00) % 0x1_00_00;
assert.notStrictEqual(
stored,
0,
@@ -167,6 +192,7 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
branchIndex + packedKeySlots + branches.length,
"Did not reserve enough space",
);
+ const dictEnd = branchIndex + packedKeySlots + branches.length;
for (const [index, [value, child]] of branches.entries()) {
assert.ok(value < 128, "Branch value too large");
const packedIndex = branchIndex + (index >> 1);
@@ -178,9 +204,12 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
"Should have the placeholder as the destination element",
);
const offset = encodeNode(child);
- // Store relative offset (pointer-position-relative).
- enc[destinationIndex] =
- (offset - destinationIndex + 0x1_00_00) % 0x1_00_00;
+ /*
+ * Store the offset relative to the end of the branch data (see
+ * the jump-table case above for why end-relative compresses
+ * better than position-relative).
+ */
+ enc[destinationIndex] = (offset - dictEnd + 0x1_00_00) % 0x1_00_00;
}
}
@@ -191,5 +220,16 @@ export function encodeTrie(trie: TrieNode, maxJumpTableOverhead = 2): number[] {
),
"Too many bits",
);
+ /*
+ * Branch pointers are stored end-relative modulo 2^16 and the decoder
+ * reconstructs them with `& 0xffff`. That round-trips only while every
+ * node index fits in a uint16; a larger trie would alias a forward
+ * pointer onto the wrong node without any per-pointer check catching it.
+ */
+ assert.ok(
+ enc.length <= 0x1_00_00,
+ `Trie has ${enc.length} words; end-relative branch pointers are ` +
+ "uint16, so the trie must not exceed 65536 words.",
+ );
return enc;
}
diff --git a/scripts/write-decode-map.ts b/scripts/write-decode-map.ts
index 86932edf..6d2a4a39 100644
--- a/scripts/write-decode-map.ts
+++ b/scripts/write-decode-map.ts
@@ -1,211 +1,34 @@
+import * as assert from "node:assert";
import * as fs from "node:fs";
import entityMap from "../maps/entities.json" with { type: "json" };
+import html4Names from "../maps/html4.json" with { type: "json" };
import legacyMap from "../maps/legacy.json" with { type: "json" };
import xmlMap from "../maps/xml.json" with { type: "json" };
+import { BinTrieFlags } from "../src/internal/bin-trie-flags.js";
+import { type EncodedTrie, encodeFullTrie } from "./trie/encode-dict.js";
import { encodeTrie } from "./trie/encode-trie.js";
-import { getTrie } from "./trie/trie.js";
-
-/**
- * Printable ASCII chars safe in JS string literals (0x21–0x7E minus `"`, `$`, `\`).
- * 91 chars. `$` is excluded to prevent `${` sequences that trip linters.
+import { getTrie, type TrieNode } from "./trie/trie.js";
+
+/*
+ * Entities defined in HTML 4.01 (lat1, symbol, and special DTDs), from
+ * maps/html4.json. These are the entities with decades of real-world usage
+ * behind them — used as the "hot" set for trie encoding decisions, so their
+ * lookup paths keep the fast jump-table encoding while the long tail of
+ * rarely-used HTML5 names can use the more compact dictionary encoding.
*/
-const SAFE: number[] = [];
-for (let codePoint = 0x21; codePoint <= 0x7e; codePoint++) {
- if (codePoint !== 0x22 && codePoint !== 0x24 && codePoint !== 0x5c) {
- SAFE.push(codePoint);
- }
-}
-const BASE = SAFE.length; // 91
-
-/** Number of most-frequent values assigned to 1-char codes. */
-const DICT_SIZE = 61;
-
-/**
- * Encode trie data using dictionary + delta-encoded value table.
- *
- * Format: [dict1: delta/RLE var-len][dict2: delta/RLE var-len][data]
- *
- * - dict1: base values for the D most-frequent entries, delta+RLE variable-length base-91.
- * - dict2: all remaining unique values, delta+RLE variable-length base-91.
- * Deltas < 90 → 1 char; 90–8279 → escape + 2 chars; larger → double-escape + 3 chars.
- * - data: each trie value encoded as 1 char (dict1 lookup) or 2 chars (dict2 lookup).
- *
- * This gives ~24% smaller raw and ~16% better gzip than base64.
- * @param data Trie data to encode.
+const HTML4_NAMES: string[] = html4Names;
+
+/*
+ * Staleness guard for `BPE_RANK_OVERRIDES` (scripts/trie/encode-dict.ts).
+ * The overrides were tuned by coordinate descent against the exact HTML trie
+ * contents, so they are stale the moment the trie changes. If this assert
+ * fires: the encoding still round-trips correctly, but the overrides (and
+ * this constant) should be re-tuned for the new data — re-run the tuning,
+ * or reset the overrides to `{}` and compare bundle gzip/brotli sizes.
*/
-function encodeTrieData(data: Uint16Array): {
- encoded: string;
- headerLength: number;
-} {
- // For small tries (e.g. XML), skip the dictionary and use plain var-len base91.
- if (data.length < 100) {
- const twoCharCount = 84;
- const split = twoCharCount * BASE;
- let result = "";
- for (const value of data) {
- if (value < split) {
- result += String.fromCharCode(
- SAFE[Math.floor(value / BASE)],
- SAFE[value % BASE],
- );
- } else {
- const adjusted = value - split;
- result += String.fromCharCode(
- SAFE[twoCharCount + Math.floor(adjusted / (BASE * BASE))],
- SAFE[Math.floor(adjusted / BASE) % BASE],
- SAFE[adjusted % BASE],
- );
- }
- }
- return { encoded: result, headerLength: 0 };
- }
-
- // Count frequencies
- const freq = new Map();
- for (const value of data) freq.set(value, (freq.get(value) ?? 0) + 1);
- // @ts-expect-error `toSorted` requires a lib bump.
- const sorted: [number, number][] = [...freq].toSorted(
- (a: [number, number], b: [number, number]) => b[1] - a[1],
- );
+const EXPECTED_HTML_ENCODED_LENGTH = 18_575;
- /*
- * Dict2 has (BASE - DICT_SIZE) * BASE 2-char codes available. The decoder
- * assumes exactly DICT_SIZE entries in dict1. Bail out loudly if the trie
- * cardinality is outside the encodable range so we never silently emit a
- * table that decodeTrieDict can't read back.
- */
- const dict2Capacity = (BASE - DICT_SIZE) * BASE;
- if (
- sorted.length < DICT_SIZE ||
- sorted.length > DICT_SIZE + dict2Capacity
- ) {
- throw new Error(
- `Trie has ${sorted.length} unique values; encoder requires [${DICT_SIZE}, ${DICT_SIZE + dict2Capacity}].`,
- );
- }
-
- // Dict1: top D values → 1-char codes, sorted ascending for delta encoding
- const dict1 = sorted
- .slice(0, DICT_SIZE)
- .map(([value]) => value)
- // eslint-disable-next-line unicorn/no-array-sort -- TS doesn't know toSorted
- .sort((a: number, b: number) => a - b);
- const dict1Set = new Set(dict1);
-
- // Dict2: remaining values, sorted ascending for delta encoding
- const dict2Sorted = sorted
- .filter(([value]: [number, number]) => !dict1Set.has(value))
- .map(([value]: [number, number]) => value)
- // eslint-disable-next-line unicorn/no-array-sort -- TS doesn't know toSorted
- .sort((a: number, b: number) => a - b);
-
- /*
- * Encode header: dict1 then dict2, each delta variable-length from 0.
- *
- * Encoding:
- * delta < 89 → 1 char: SAFE[delta]
- * SAFE[89] → run-length marker: next char encodes N-2 (≥1),
- * meaning N consecutive delta-1 values
- * SAFE[90] → escape for large deltas (same as before but threshold 89)
- * SAFE[90] SAFE[90] → double escape for very large deltas
- */
- const RLE_MARKER = SAFE[89];
- const ESCAPE = SAFE[90];
- let header = "";
- function deltaEncode(values: number[]) {
- let previous = 0;
- let index = 0;
- while (index < values.length) {
- const delta = values[index] - previous;
- if (delta === 1) {
- // Count consecutive delta=1 values
- let runLength = 1;
- while (
- index + runLength < values.length &&
- values[index + runLength] -
- values[index + runLength - 1] ===
- 1
- ) {
- runLength++;
- }
- if (runLength >= 3) {
- // Emit RLE-encoded runs (max chunk = BASE+1=92, stored as SAFE[0..90])
- let remaining = runLength;
- while (remaining >= 3) {
- const chunk = Math.min(remaining, BASE + 1);
- header += String.fromCharCode(
- RLE_MARKER,
- SAFE[chunk - 2],
- );
- remaining -= chunk;
- }
- // Emit leftover 1-2 values as plain delta=1
- for (let r = 0; r < remaining; r++) {
- header += String.fromCharCode(SAFE[1]);
- }
- previous = values[index + runLength - 1];
- index += runLength;
- continue;
- }
- }
- // Non-run or short run: emit single delta
- if (delta < 89) {
- header += String.fromCharCode(SAFE[delta]);
- } else {
- const adjusted = delta - 89;
- header +=
- adjusted < 90 * BASE
- ? String.fromCharCode(
- ESCAPE,
- SAFE[Math.floor(adjusted / BASE)],
- SAFE[adjusted % BASE],
- )
- : String.fromCharCode(
- ESCAPE,
- ESCAPE,
- SAFE[Math.floor(adjusted / (BASE * BASE))],
- SAFE[Math.floor(adjusted / BASE) % BASE],
- SAFE[adjusted % BASE],
- );
- }
- previous = values[index];
- index++;
- }
- }
- deltaEncode(dict1);
- deltaEncode(dict2Sorted);
-
- // Build value → code mapping
- const valueToCode = new Map();
- for (let index = 0; index < DICT_SIZE; index++) {
- valueToCode.set(dict1[index], String.fromCharCode(SAFE[index]));
- }
- let codeIndex = 0;
- for (const value of dict2Sorted) {
- valueToCode.set(
- value,
- String.fromCharCode(
- SAFE[DICT_SIZE + Math.floor(codeIndex / BASE)],
- SAFE[codeIndex % BASE],
- ),
- );
- codeIndex++;
- }
-
- // Encode data
- let encodedData = "";
- for (const value of data) {
- const code = valueToCode.get(value);
- if (code === undefined) {
- throw new Error(
- `No encoded code assigned for trie value ${value}.`,
- );
- }
- encodedData += code;
- }
-
- return { encoded: header + encodedData, headerLength: header.length };
-}
+// --- File generation ------------------------------------------------------
function formatNumber(value: number): string {
return value >= 10_000
@@ -213,41 +36,166 @@ function formatNumber(value: number): string {
: String(value);
}
-function generateFile(name: string, data: Uint16Array): string {
- const { encoded, headerLength } = encodeTrieData(data);
+/**
+ * Formatter line width — must match biome's configured width (the default,
+ * 80) so `biome check` leaves the generated files untouched.
+ */
+const FORMAT_LINE_WIDTH = 80;
+/** Max content chars per line: width minus 4-space indent and trailing comma. */
+const FORMAT_CONTENT_WIDTH = FORMAT_LINE_WIDTH - 4 - 1;
- // For small tries, emit an inline literal array (no decoder import needed).
- if (headerLength === 0) {
- const values = [...data].map((v) => formatNumber(v)).join(", ");
- return `// Generated using scripts/write-decode-map.ts
+function generateInlineFile(name: string, data: Uint16Array): string {
+ /*
+ * Greedily fill lines to the formatter's width, matching biome's array
+ * formatting so the formatter leaves the generated file untouched.
+ */
+ const tokens = [...data].map((v) => formatNumber(v));
+ const lines: string[] = [];
+ let line = "";
+ for (const token of tokens) {
+ const piece = (line ? ", " : "") + token;
+ if (line && line.length + piece.length > FORMAT_CONTENT_WIDTH) {
+ lines.push(`${line},`);
+ line = token;
+ } else {
+ line += piece;
+ }
+ }
+ if (line) lines.push(`${line},`);
+ const body = lines.map((l) => ` ${l}`).join("\n");
+ return `// Generated using scripts/write-decode-map.ts
/** Packed ${name.toUpperCase()} decode trie data. */
export const ${name}DecodeTree: Uint16Array = /* #__PURE__ */ new Uint16Array([
- ${values},
+${body}
]);`;
- }
+}
+function generateDecoderFile(
+ name: string,
+ data: Uint16Array,
+ result: EncodedTrie,
+): string {
return `// Generated using scripts/write-decode-map.ts
import { decodeTrieDict } from "../internal/decode-shared.js";
/** Packed ${name.toUpperCase()} decode trie data. */
export const ${name}DecodeTree: Uint16Array = /* #__PURE__ */ decodeTrieDict(
- ${JSON.stringify(encoded)},
+ ${JSON.stringify(result.encoded)},
${formatNumber(data.length)},
- ${formatNumber(headerLength)},
+ ${formatNumber(result.atomCount)},
+ ${formatNumber(result.dict1AtomCount)},
+ ${formatNumber(result.ngramCount)},
+ ${result.dictSize},
);`;
}
+/**
+ * Count how many entities pass through each trie node (node "traffic").
+ * Shared (deduplicated) subtree nodes accumulate counts from every path
+ * that reaches them.
+ * @param root The trie root.
+ * @param keys The entity names inserted into the trie.
+ */
+function computeNodeTraffic(
+ root: TrieNode,
+ keys: string[],
+): Map {
+ const traffic = new Map([[root, keys.length]]);
+ for (const key of keys) {
+ let node = root;
+ for (let index = 0; index < key.length; index++) {
+ const next = node.next?.get(key.charCodeAt(index));
+ // eslint-disable-next-line unicorn/no-break-in-nested-loop
+ if (!next) break;
+ node = next;
+ traffic.set(node, (traffic.get(node) ?? 0) + 1);
+ }
+ }
+ return traffic;
+}
+
function convertMapToBinaryTrie(
name: "html" | "xml",
map: Record,
legacy: Record,
) {
- const encoded = new Uint16Array(encodeTrie(getTrie(map, legacy), 1.2));
- const code = `${generateFile(name, encoded)}\n`;
+ /*
+ * Hot/cold jump-table threshold: nodes on the lookup path of an HTML4
+ * entity (the empirically common set) or with high entity traffic keep
+ * `maxJumpTableOverhead=4` (jump tables: O(1) indexed read, handled
+ * inline by the decoder's descent loop — −22% to −30% decode time on
+ * entity-dense workloads). The long tail of rare HTML5 names uses the
+ * compact linear-scan dictionary encoding instead, which keeps the
+ * trie words (and the shipped bundle) smaller.
+ */
+ const hotTraffic = 16;
+ const coldOverhead = 1.2;
+ const trie = getTrie(map, legacy);
+ const hotNodes = new Set();
+ for (const name of HTML4_NAMES) {
+ let node: TrieNode | undefined = trie;
+ hotNodes.add(node);
+ for (let index = 0; index < name.length && node; index++) {
+ node = node.next?.get(name.charCodeAt(index));
+ if (node) hotNodes.add(node);
+ }
+ }
+ const traffic = computeNodeTraffic(trie, Object.keys(map));
+ const data = new Uint16Array(
+ encodeTrie(trie, (node) =>
+ hotNodes.has(node) || (traffic.get(node) ?? 0) >= hotTraffic
+ ? 4
+ : coldOverhead,
+ ),
+ );
+
+ /*
+ * `decodeWithTrie` (used for all HTML decoding) inlines root navigation
+ * assuming the root header is a multi-branch jump table — it falls back
+ * to rejecting every entity, not to a slow path, if the shape differs.
+ * Fail the build instead of shipping a trie that silently never
+ * matches. (The XML trie is exempt: `decodeXML` has a hand-coded fast
+ * path and the streaming decoder handles any root shape.)
+ */
+ const rootJumpOffset = data[0] & BinTrieFlags.JUMP_TABLE;
+ const rootBranchCount = (data[0] & BinTrieFlags.BRANCH_LENGTH) >> 7;
+ /*
+ * The decoder's inline root navigation also assumes the root carries no
+ * value and is not a compact run; otherwise the descent loop is skipped
+ * and every entity is rejected.
+ */
+ const hasRootValueOrRun =
+ (data[0] & (BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)) !== 0;
+ if (
+ name === "html" &&
+ (rootJumpOffset === 0 || rootBranchCount === 0 || hasRootValueOrRun)
+ ) {
+ throw new Error(
+ "HTML trie root must be a value-less multi-branch jump table for " +
+ "the decoder's inline root navigation; got header " +
+ `0x${data[0].toString(16)}.`,
+ );
+ }
+
+ let file: string;
+ if (data.length < 100) {
+ // Tiny tries (XML) skip the dict; ~25 values fits inline cheaply.
+ file = generateInlineFile(name, data);
+ } else {
+ const result = encodeFullTrie(data);
+ assert.strictEqual(
+ result.encoded.length,
+ EXPECTED_HTML_ENCODED_LENGTH,
+ "Encoded HTML trie length changed — BPE_RANK_OVERRIDES (and " +
+ "EXPECTED_HTML_ENCODED_LENGTH) are stale; see the comment " +
+ "on the constant.",
+ );
+ file = generateDecoderFile(name, data, result);
+ }
fs.writeFileSync(
new URL(`../src/generated/decode-data-${name}.ts`, import.meta.url),
- code,
+ `${file}\n`,
);
}
diff --git a/src/decode-codepoint.ts b/src/decode-codepoint.ts
index 570a0ad8..82dcfe10 100644
--- a/src/decode-codepoint.ts
+++ b/src/decode-codepoint.ts
@@ -32,3 +32,19 @@ export function replaceCodePoint(codePoint: number): number {
return codePoint;
}
+
+/**
+ * Convert the code point of a decoded numeric entity to a string, replacing
+ * invalid values.
+ *
+ * Fast path for plain BMP code points: [1..0x7F] and [0xA0..0xD7FF] pass
+ * `replaceCodePoint` unchanged (no NUL, C1 remap, surrogate, or out-of-range
+ * handling) and fit a single charCode. 0xd760 = 0xD800 (the first surrogate)
+ * - 0xA0.
+ * @param codePoint Unicode code point to convert.
+ */
+export function codePointToString(codePoint: number): string {
+ return (codePoint - 1) >>> 0 < 0x7f || (codePoint - 0xa0) >>> 0 < 0xd7_60
+ ? String.fromCharCode(codePoint)
+ : String.fromCodePoint(replaceCodePoint(codePoint));
+}
diff --git a/src/decode-stream.spec.ts b/src/decode-stream.spec.ts
index 26078c55..1f53827e 100644
--- a/src/decode-stream.spec.ts
+++ b/src/decode-stream.spec.ts
@@ -1,8 +1,39 @@
import { describe, expect, it, vi } from "vitest";
-import { DecodingMode, EntityDecoder } from "./decode.js";
+import entityMap from "../maps/entities.json" with { type: "json" };
+import xmlMap from "../maps/xml.json" with { type: "json" };
+import {
+ DecodingMode,
+ decodeHTML,
+ decodeXML,
+ EntityDecoder,
+} from "./decode.js";
import { htmlDecodeTree } from "./generated/decode-data-html.js";
import { xmlDecodeTree } from "./generated/decode-data-xml.js";
+/**
+ * Decode `&name;` through EntityDecoder in `chunkSize`-char writes.
+ * @param decodeTree The trie to decode against.
+ * @param input Full entity text, starting at the `&`.
+ * @param chunkSize Characters per write call.
+ */
+function streamEntity(
+ decodeTree: Uint16Array,
+ input: string,
+ chunkSize: number,
+): { output: string; consumed: number } {
+ let output = "";
+ const decoder = new EntityDecoder(decodeTree, (cp) => {
+ output += String.fromCodePoint(cp);
+ });
+ decoder.startEntity(DecodingMode.Legacy);
+ let consumed = -1;
+ for (let pos = 1; pos < input.length && consumed < 0; pos += chunkSize) {
+ consumed = decoder.write(input.slice(pos, pos + chunkSize), 0);
+ }
+ if (consumed < 0) consumed = decoder.end();
+ return { output, consumed };
+}
+
describe("EntityDecoder Streaming", () => {
it("should decode long entities split across chunks (char-by-char)", () => {
const callback = vi.fn();
@@ -152,4 +183,155 @@ describe("EntityDecoder Streaming", () => {
expect(callback).toHaveBeenCalledTimes(5);
});
+
+ /*
+ * A legacy entity ending in a compact run (`Á` — "cute" is a run)
+ * must report exactly the entity's length as consumed (7, not 8): the
+ * run's final character is part of the match, not excess. One extra
+ * consumed character here makes a streaming parser swallow the
+ * character following the entity.
+ */
+ describe("consumed count for legacy entities ending in a compact run", () => {
+ const entity = "Á"; // 7 chars; "cute" is a compact run.
+ const codepoint = 0xc1; // Á
+
+ it("should report 7 consumed when terminated by another char", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ expect(decoder.write(`${entity} x`, 1)).toBe(entity.length);
+ expect(callback).toHaveBeenCalledWith(codepoint, entity.length);
+ });
+
+ it("should report 7 consumed at the end of input", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ expect(decoder.write(entity, 1)).toBe(-1);
+ expect(decoder.end()).toBe(entity.length);
+ expect(callback).toHaveBeenCalledWith(codepoint, entity.length);
+ });
+
+ it("should report 7 consumed when written char-by-char", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ for (let index = 1; index < entity.length; index++) {
+ expect(decoder.write(entity[index], 0)).toBe(-1);
+ }
+ expect(decoder.write(" ", 0)).toBe(entity.length);
+ expect(callback).toHaveBeenCalledWith(codepoint, entity.length);
+ });
+
+ it("should still include the semicolon when present", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ expect(decoder.write(`${entity};`, 1)).toBe(entity.length + 1);
+ expect(callback).toHaveBeenCalledWith(codepoint, entity.length + 1);
+ });
+
+ it("should reject in attribute mode when followed by `=`", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Attribute);
+ expect(decoder.write(`${entity}=`, 1)).toBe(0);
+ expect(callback).not.toHaveBeenCalled();
+ });
+ });
+
+ /*
+ * Exhaustive writer↔reader agreement for the streaming descent: it is
+ * the only trie reader without a full-map test otherwise. Every entity
+ * from both maps goes through EntityDecoder whole and char-by-char, and
+ * must match the sync decoders in both output and consumed count.
+ */
+ describe("exhaustive full-map agreement with the sync decoders", () => {
+ const chunkSizes = [
+ ["whole", Number.MAX_SAFE_INTEGER],
+ ["char-by-char", 1],
+ ] as const;
+
+ it.each(
+ chunkSizes,
+ )("should decode every HTML entity (%s)", (_name, chunkSize) => {
+ for (const name of Object.keys(entityMap)) {
+ const input = `&${name};`;
+ const result = streamEntity(htmlDecodeTree, input, chunkSize);
+ expect(result.output).toBe(decodeHTML(input));
+ expect(result.consumed).toBe(input.length);
+ }
+ });
+
+ it.each(
+ chunkSizes,
+ )("should decode every XML entity (%s)", (_name, chunkSize) => {
+ for (const name of Object.keys(xmlMap)) {
+ const input = `&${name};`;
+ const result = streamEntity(xmlDecodeTree, input, chunkSize);
+ expect(result.output).toBe(decodeXML(input));
+ expect(result.consumed).toBe(input.length);
+ }
+ });
+ });
+
+ /*
+ * Chunk-boundary invariants of the resumable walk: a legacy match that
+ * lands exactly on a chunk boundary must be recorded before the chunk
+ * ends, so a subsequent `end()` (or rejection in the next chunk) emits
+ * it with the right consumed count.
+ */
+ describe("legacy matches at chunk boundaries", () => {
+ it("should emit a match reached mid-descent across chunks via end()", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ expect(decoder.write("no", 0)).toBe(-1);
+ expect(decoder.write("t", 0)).toBe(-1);
+ expect(decoder.end()).toBe(4);
+ expect(callback).toHaveBeenCalledWith(0xac, 4); // ¬
+ });
+
+ it("should emit a compact-run match split across chunks via end()", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Legacy);
+ expect(decoder.write("Aac", 0)).toBe(-1);
+ expect(decoder.write("ute", 0)).toBe(-1);
+ expect(decoder.end()).toBe(7);
+ expect(callback).toHaveBeenCalledWith(0xc1, 7); // Á
+ });
+
+ it("should not record strict-only matches at a chunk end", () => {
+ const callback = vi.fn();
+ const decoder = new EntityDecoder(htmlDecodeTree, callback);
+
+ decoder.startEntity(DecodingMode.Strict);
+ expect(decoder.write("amp", 0)).toBe(-1);
+ expect(decoder.end()).toBe(0);
+ expect(callback).not.toHaveBeenCalled();
+ });
+
+ it("should apply attribute terminator rules across a chunk boundary", () => {
+ const callback = vi.fn();
+ const rejecting = new EntityDecoder(htmlDecodeTree, callback);
+ rejecting.startEntity(DecodingMode.Attribute);
+ expect(rejecting.write("Aacute", 0)).toBe(-1);
+ expect(rejecting.write("=", 0)).toBe(0);
+ expect(callback).not.toHaveBeenCalled();
+
+ const accepting = new EntityDecoder(htmlDecodeTree, callback);
+ accepting.startEntity(DecodingMode.Attribute);
+ expect(accepting.write("Aacute", 0)).toBe(-1);
+ expect(accepting.write(" ", 0)).toBe(7);
+ expect(callback).toHaveBeenCalledWith(0xc1, 7);
+ });
+ });
});
diff --git a/src/decode.spec.ts b/src/decode.spec.ts
index a36519a6..0fbda4d9 100644
--- a/src/decode.spec.ts
+++ b/src/decode.spec.ts
@@ -1,4 +1,8 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import entityMap from "../maps/entities.json" with { type: "json" };
+import html4Names from "../maps/html4.json" with { type: "json" };
+import legacyMap from "../maps/legacy.json" with { type: "json" };
+import xmlMap from "../maps/xml.json" with { type: "json" };
import * as entities from "./decode.js";
/**
@@ -180,6 +184,58 @@ describe.each(implementations)("Decode test: %s", (_name, {
});
});
+ describe("overlong numeric entities (packed consumed-field overflow)", () => {
+ /*
+ * The sync `parseNumericEntity` packs `(consumed << 21) | cp`;
+ * entities of ≥ 2047 characters exceed the 11-bit consumed field
+ * and take the `longNumericConsumed` side channel. Whatever the
+ * body length, the whole entity must be consumed and produce
+ * U+FFFD (matching entities@7.0.1) — a wrapped consumed count
+ * would leak digits into the output.
+ */
+ const digitCounts = [2045, 2046, 2047, 2048, 4096];
+ const bases: [string, string][] = [
+ ["decimal", ""],
+ ["hex", "x"],
+ ];
+
+ describe.each(bases)("%s", (_base, prefix) => {
+ it.each(
+ digitCounts,
+ )("should decode a %i-digit body with semicolon", (count) => {
+ const input = `&#${prefix}${"1".repeat(count)};X`;
+ expect(decodeHTML(input)).toBe("�X");
+ expect(decodeHTMLStrict(input)).toBe("�X");
+ expect(decodeHTMLAttribute(input)).toBe("�X");
+ expect(decodeXML(input)).toBe("�X");
+ });
+
+ it.each(
+ digitCounts,
+ )("should decode a %i-digit body without semicolon", (count) => {
+ const input = `&#${prefix}${"1".repeat(count)}X`;
+ expect(decodeHTML(input)).toBe("�X");
+ expect(decodeHTMLAttribute(input)).toBe("�X");
+ // Strict modes require the semicolon.
+ expect(decodeHTMLStrict(input)).toBe(input);
+ expect(decodeXML(input)).toBe(input);
+ });
+
+ it.each(
+ digitCounts,
+ )("should decode a %i-digit body at the end of input", (count) => {
+ const input = `&#${prefix}${"1".repeat(count)}`;
+ expect(decodeHTML(input)).toBe("�");
+ expect(decodeHTMLStrict(input)).toBe(input);
+ });
+ });
+
+ it("should recover the exact code point after thousands of leading zeros", () => {
+ expect(decodeHTML(`&#${"0".repeat(2048)}65;X`)).toBe("AX");
+ expect(decodeXML(`&#x${"0".repeat(2048)}41;X`)).toBe("AX");
+ });
+ });
+
it("should parse followed by < (#852)", () =>
expect(decodeHTML(" <")).toBe("\u{A0}<"));
@@ -223,8 +279,8 @@ describe.each(implementations)("Decode test: %s", (_name, {
* - numeric entities are always accepted
* - leaf-node legacy match (e.g. `amp`) followed by a char that
* isn't an invalid attribute terminator. The trailing char
- * equals the entity's value byte, which previously sent the
- * streaming decoder into a phantom node.
+ * equals the entity's value byte — the decoder must not read
+ * the value slot as a trie node and descend into it.
*/
const acceptCases = [
{ input: "¬", output: "¬" },
@@ -246,6 +302,100 @@ describe.each(implementations)("Decode test: %s", (_name, {
output,
}) => expect(decodeHTMLAttribute(input)).toBe(output));
});
+
+ describe("full entity maps (regression guard for trie generation)", () => {
+ it("should decode every named entity from the WHATWG map", () => {
+ for (const [name, value] of Object.entries(entityMap)) {
+ expect(decodeHTML(`&${name};`)).toBe(value);
+ expect(decodeHTMLStrict(`&${name};`)).toBe(value);
+ }
+ });
+
+ it("should decode every XML entity", () => {
+ for (const [name, value] of Object.entries(xmlMap)) {
+ expect(decodeXML(`&${name};`)).toBe(value);
+ }
+ });
+
+ /*
+ * Covers the streaming `consumed` bookkeeping for entities ending in
+ * compact trie runs: a wrong consumed count makes the streaming
+ * implementations drop or duplicate characters around the entity.
+ */
+ it("should decode every legacy entity without a semicolon", () => {
+ for (const [name, value] of Object.entries(legacyMap)) {
+ expect(decodeHTML(`&${name}`)).toBe(value);
+ expect(decodeHTML(`&${name} after`)).toBe(`${value} after`);
+ expect(decodeHTML(`x&${name}-y`)).toBe(`x${value}-y`);
+ }
+ });
+ });
+
+ describe("non-entities with legacy-like prefixes stay literal", () => {
+ /*
+ * In entities <= 7.0.1, a failed named-entity match could read the
+ * legacy result from a wrong trie index, emitting an unrelated
+ * character (e.g. `&Gdot ` → `Â`). These inputs must stay literal.
+ */
+ const literalCases = [
+ "&Gdot ",
+ "&eta=",
+ "&Ocy ",
+ "&YUcy1",
+ "&backepsilonx",
+ "&bepsix",
+ ];
+
+ it.each(literalCases)("should not decode %j", (input) => {
+ expect(decodeHTML(input)).toBe(input);
+ expect(decodeHTMLStrict(input)).toBe(input);
+ expect(decodeHTMLAttribute(input)).toBe(input);
+ });
+
+ /*
+ * Legacy prefixes of longer names decode in text mode but must stay
+ * literal in attribute mode (next char is alphanumeric).
+ */
+ const prefixCases = [
+ { input: "¢erdot ", text: "¢erdot " },
+ { input: "©sr ", text: "©sr " },
+ { input: "÷ontimes ", text: "÷ontimes " },
+ { input: ">cc ", text: ">cc " },
+ ];
+
+ it.each(
+ prefixCases,
+ )("should decode $input as legacy prefix only in text mode", ({
+ input,
+ text,
+ }) => {
+ expect(decodeHTML(input)).toBe(text);
+ expect(decodeHTMLAttribute(input)).toBe(input);
+ });
+ });
+});
+
+/*
+ * `maps/html4.json` drives the hot/cold trie-encoding split in
+ * `scripts/write-decode-map.ts`: names listed there keep the fast jump-table
+ * encoding. A name that is duplicated (wasted hot budget) or missing from the
+ * WHATWG map (silently ignored when marking hot paths) would degrade the
+ * generated trie without failing the build.
+ */
+describe("entity map contracts", () => {
+ it("html4.json should be duplicate-free", () => {
+ const duplicates = html4Names.filter(
+ (name, index) => html4Names.indexOf(name) !== index,
+ );
+ expect(duplicates).toStrictEqual([]);
+ });
+
+ it("every html4.json name should be a key in entities.json", () => {
+ const missing = html4Names.filter(
+ (name) => !Object.hasOwn(entityMap, name),
+ );
+ expect(missing).toStrictEqual([]);
+ });
});
describe("EntityDecoder", () => {
@@ -351,6 +501,57 @@ describe("EntityDecoder", () => {
* Discovered prefix: "zi" followed by compact run "grarr"; mismatching inside this run should
* return 0 with no emission (result still 0).
*/
+ describe("overlong numeric entities", () => {
+ /*
+ * The streaming decoder tracks `consumed` as a plain field, so —
+ * unlike the sync parser's packed return value — no length ever
+ * overflows. These pin the equivalence for the sync boundary cases.
+ */
+ const digitCounts = [2045, 2046, 2047, 2048, 4096];
+
+ it.each(
+ digitCounts,
+ )("should report full consumed for %i decimal digits", (count) => {
+ decoder.startEntity(entities.DecodingMode.Strict);
+ expect(decoder.write(`&#${"1".repeat(count)};`, 1)).toBe(count + 3);
+ expect(callback).toHaveBeenCalledExactlyOnceWith(
+ 0xff_fd,
+ count + 3,
+ );
+ });
+
+ it.each(
+ digitCounts,
+ )("should report full consumed for %i hex digits", (count) => {
+ decoder.startEntity(entities.DecodingMode.Strict);
+ expect(decoder.write(`&#x${"1".repeat(count)};`, 1)).toBe(
+ count + 4,
+ );
+ expect(callback).toHaveBeenCalledExactlyOnceWith(
+ 0xff_fd,
+ count + 4,
+ );
+ });
+
+ it("should clamp the accumulator before it reaches the errors callbacks", () => {
+ const errorHandlers = {
+ missingSemicolonAfterCharacterReference: vi.fn(),
+ absenceOfDigitsInNumericCharacterReference: vi.fn(),
+ validateNumericCharacterReference: vi.fn(),
+ };
+ const errorDecoder = new entities.EntityDecoder(
+ entities.htmlDecodeTree,
+ callback,
+ errorHandlers,
+ );
+ errorDecoder.startEntity(entities.DecodingMode.Strict);
+ errorDecoder.write(`&#x${"f".repeat(4096)};`, 1);
+ expect(
+ errorHandlers.validateNumericCharacterReference,
+ ).toHaveBeenCalledExactlyOnceWith(0x11_00_00);
+ });
+ });
+
describe("compact run mismatches", () => {
it.each([
["first run character mismatch", "ziXgrar"],
diff --git a/src/decode.ts b/src/decode.ts
index 849bf686..dd7de1ae 100644
--- a/src/decode.ts
+++ b/src/decode.ts
@@ -1,8 +1,9 @@
-import { replaceCodePoint } from "./decode-codepoint.js";
+import { codePointToString, replaceCodePoint } from "./decode-codepoint.js";
import { htmlDecodeTree } from "./generated/decode-data-html.js";
import { BinTrieFlags } from "./internal/bin-trie-flags.js";
const enum CharCodes {
+ AMP = 38, // "&"
NUM = 35, // "#"
SEMI = 59, // ";"
EQUALS = 61, // "="
@@ -21,6 +22,35 @@ const enum CharCodes {
/** Bit that needs to be set to convert an upper case ASCII character to lower case */
const TO_LOWER_BIT = 0b10_0000;
+/*
+ * `parseNumericEntity` packs its two results into one 32-bit integer:
+ * `(consumed << CONSUMED_SHIFT) | codePoint`. The 21-bit code point field
+ * fits any valid Unicode value (max 0x10FFFF, clamped before packing); the
+ * consumed count gets the remaining 11 bits. Extract `consumed` with `>>>`
+ * so the topmost bit isn't treated as a sign.
+ *
+ * Plain consts rather than a `const enum`: with `isolatedModules`, enum
+ * member reads compile to runtime property loads, which showed up in
+ * numeric-entity-heavy benchmarks on these per-entity paths.
+ */
+const CONSUMED_SHIFT = 21;
+const CODE_POINT_MASK = 0x1f_ff_ff;
+/**
+ * All-ones consumed field: the entity is at least 2047 characters long
+ * and the true count is in `longNumericConsumed`.
+ */
+const CONSUMED_OVERFLOW = 0x7_ff;
+
+/**
+ * Side channel for the rare numeric entity whose length doesn't fit the
+ * packed 11-bit consumed field (entities of ≥ 2047 characters, i.e. bodies
+ * of ~2045+ digits). Set by `parseNumericEntity` whenever the consumed
+ * field it returns is `CONSUMED_OVERFLOW`; callers read the true length
+ * from here. A module-level slot avoids a tuple allocation on the hot path
+ * for an input that in practice only appears in fuzzers and attacks.
+ */
+let longNumericConsumed = 0;
+
/**
* Unsigned subtraction trick: (code - lo) >>> 0 wraps negatives to large
* values, so a single `<=` covers the entire [lo..hi] range check.
@@ -91,18 +121,23 @@ export class EntityDecoder {
/**
* The result of the entity.
*
- * Either the result index of a named entity, or the codepoint of a
- * numeric entity.
+ * For named entities: the trie index of the best legacy match so far
+ * (0 = none). For numeric entities: the accumulated code point.
*/
private result = 0;
/** The current index in the decode tree. */
private treeIndex = 0;
- /** The number of characters that were consumed in excess. */
+ /**
+ * Characters consumed since the last recorded legacy match, plus one.
+ * Invariant at the top of the `stateNamedEntity` loop: `excess` equals
+ * the number of unrecorded consumed characters + 1.
+ */
private excess = 1;
/** The mode in which the decoder is operating. */
private decodeMode = DecodingMode.Strict;
/** The number of characters that have been consumed in the current run. */
+ // biome-ignore lint/correctness/noUnusedPrivateClassMembers: False positive
private runConsumed = 0;
constructor(
@@ -141,7 +176,7 @@ export class EntityDecoder {
* Write an entity to the decoder. This can be called multiple times with partial entities.
* If the entity is incomplete, the decoder will return -1.
*
- * Mirrors the implementation of `getDecoder`, but with the ability to stop decoding if the
+ * Mirrors the non-streaming `decodeWithTrie`, but with the ability to stop decoding if the
* entity is incomplete, and resume when the next string is written.
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The offset at which the entity begins. Should be 0 if this is not the first call.
@@ -205,13 +240,19 @@ export class EntityDecoder {
/**
* Parses a hexadecimal numeric entity.
*
- * Equivalent to the `Hexademical character reference state` in the HTML spec.
+ * Equivalent to the `Hexademical character reference state` in the HTML
+ * spec. Sync counterpart: the hex loop in `parseNumericEntity`; digit
+ * and clamping behavior must stay in sync between the two.
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNumericHex(input: string, offset: number): number {
- while (offset < input.length) {
+ const inputLength = input.length;
+ // Local accumulators; flushed before any exit (see stateNamedEntity).
+ let { result } = this;
+ let { consumed } = this;
+ while (offset < inputLength) {
const char = input.charCodeAt(offset);
if (isNumber(char) || isHexadecimalCharacter(char)) {
// Convert hex digit to value (0-15); 'a'/'A' -> 10.
@@ -219,35 +260,57 @@ export class EntityDecoder {
char <= CharCodes.NINE
? char - CharCodes.ZERO
: (char | TO_LOWER_BIT) - CharCodes.LOWER_A + 10;
- this.result = this.result * 16 + digit;
- this.consumed += 1;
+ result = result * 16 + digit;
+ /*
+ * Clamp overflow to 1 past the Unicode max, matching
+ * `parseNumericEntity`. Keeps the accumulator a small
+ * integer (long bodies would reach Infinity) so the
+ * `errors` callbacks always see a finite code point.
+ */
+ if (result > 0x10_ff_ff) result = 0x11_00_00;
+ consumed += 1;
offset += 1;
} else {
+ this.result = result;
+ this.consumed = consumed;
return this.emitNumericEntity(char, 3);
}
}
+ this.result = result;
+ this.consumed = consumed;
return -1; // Incomplete entity
}
/**
* Parses a decimal numeric entity.
*
- * Equivalent to the `Decimal character reference state` in the HTML spec.
+ * Equivalent to the `Decimal character reference state` in the HTML
+ * spec. Sync counterpart: the decimal loop in `parseNumericEntity`;
+ * digit and clamping behavior must stay in sync between the two.
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNumericDecimal(input: string, offset: number): number {
- while (offset < input.length) {
- const char = input.charCodeAt(offset);
- if (isNumber(char)) {
- this.result = this.result * 10 + (char - CharCodes.ZERO);
- this.consumed += 1;
- offset += 1;
- } else {
- return this.emitNumericEntity(char, 2);
+ const inputLength = input.length;
+ // Local accumulators; flushed before any exit (see stateNamedEntity).
+ let { result } = this;
+ let { consumed } = this;
+ while (offset < inputLength) {
+ const digit = input.charCodeAt(offset) - CharCodes.ZERO;
+ if (digit >>> 0 > 9) {
+ this.result = result;
+ this.consumed = consumed;
+ return this.emitNumericEntity(digit + CharCodes.ZERO, 2);
}
+ result = result * 10 + digit;
+ // Clamp overflow, matching `parseNumericEntity` (see stateNumericHex).
+ if (result > 0x10_ff_ff) result = 0x11_00_00;
+ consumed += 1;
+ offset += 1;
}
+ this.result = result;
+ this.consumed = consumed;
return -1; // Incomplete entity
}
@@ -311,6 +374,25 @@ export class EntityDecoder {
: this.emitNotTerminatedNamedEntity();
}
+ /**
+ * Flush locally-tracked walk state back to the fields, then emit the
+ * recorded legacy match or reject (cold path — at most once per entity).
+ * @param consumed Locally-tracked consumed count.
+ * @param excess Locally-tracked excess count.
+ * @param char Pending input character (may be the mismatching char).
+ * @param valueLength Value length at the current trie node.
+ */
+ private flushAndEmitLegacyOrReject(
+ consumed: number,
+ excess: number,
+ char: number,
+ valueLength: number,
+ ): number {
+ this.consumed = consumed;
+ this.excess = excess;
+ return this.emitLegacyOrReject(char, valueLength);
+ }
+
/**
* Parses a named entity.
*
@@ -322,138 +404,232 @@ export class EntityDecoder {
private stateNamedEntity(input: string, offset: number): number {
const { decodeTree } = this;
const inputLength = input.length;
- let current = decodeTree[this.treeIndex];
- // The number of bytes of the value, including the current byte.
- let valueLength = current >>> 14;
+ const isStrict = this.decodeMode === DecodingMode.Strict;
+
+ /*
+ * Local copies of the resumable walk state — a store per character
+ * is what made this path slow. They are flushed back to the fields
+ * on every exit (chunk end, and before any emit helper that reads
+ * them). `this.result` is only written at the (rare) record points,
+ * so it stays a direct field write.
+ *
+ * Legacy-match recording happens in two idempotent places: at the
+ * loop top when sitting on a value node, and in the chunk-end
+ * epilogue (so `end()` sees matches that land exactly on a chunk
+ * boundary). Recording applies `consumed += excess - 1; excess = 1`,
+ * which is a no-op when repeated — the loop-top invariant is
+ * `excess` = unrecorded consumed characters + 1.
+ */
+ let { treeIndex } = this;
+ let { excess } = this;
+ let { consumed } = this;
+ let current = decodeTree[treeIndex];
while (offset < inputLength) {
- // Handle compact runs (possibly resumable): valueLength == 0 and FLAG13 set.
- if (valueLength === 0 && (current & BinTrieFlags.FLAG13) !== 0) {
+ /*
+ * Descend through value-less jump-table nodes (including the
+ * single-branch encoding) inline, mirroring `decodeWithTrie`:
+ * this avoids a `determineBranch` call per level for the
+ * dominant node shape — including the root on the first write.
+ */
+ while (
+ (current &
+ (BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)) ===
+ 0 &&
+ (current & BinTrieFlags.JUMP_TABLE) !== 0
+ ) {
+ const jumpOffset = current & BinTrieFlags.JUMP_TABLE;
+ const branchCount = (current & BinTrieFlags.BRANCH_LENGTH) >> 7;
+ if (branchCount === 0) {
+ // Single branch encoded inline in the jump offset bits.
+ if (input.charCodeAt(offset) !== jumpOffset) {
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
+ input.charCodeAt(offset),
+ 0,
+ );
+ }
+ treeIndex += 1;
+ } else {
+ const slot = input.charCodeAt(offset) - jumpOffset;
+ if (slot >>> 0 >= branchCount) {
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
+ input.charCodeAt(offset),
+ 0,
+ );
+ }
+ const stored = decodeTree[treeIndex + 1 + slot];
+ if (stored === 0) {
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
+ input.charCodeAt(offset),
+ 0,
+ );
+ }
+ // End-relative: branch data ends at treeIndex+1+branchCount.
+ treeIndex = (treeIndex + branchCount + stored) & 0xff_ff;
+ }
+ current = decodeTree[treeIndex];
+ offset += 1;
+ excess += 1;
+ /*
+ * `charCodeAt` past the end returns NaN, which would alias
+ * to slot 0 after `>>> 0` — bail out explicitly.
+ */
+ // eslint-disable-next-line unicorn/no-break-in-nested-loop
+ if (offset >= inputLength) break;
+ }
+ if (offset >= inputLength) break;
+
+ // Handle compact runs (resumable across chunks).
+ if (
+ (current &
+ (BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)) ===
+ BinTrieFlags.FLAG13
+ ) {
const runLength =
- (current & BinTrieFlags.BRANCH_LENGTH) >> 7; /* 2..63 */
+ (current & BinTrieFlags.BRANCH_LENGTH) >> 7; /* 3..63 */
+ let { runConsumed } = this;
// If we are starting a run, check the first char.
- if (this.runConsumed === 0) {
+ if (runConsumed === 0) {
const firstChar = current & BinTrieFlags.JUMP_TABLE;
if (input.charCodeAt(offset) !== firstChar) {
- return this.emitLegacyOrReject(
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
input.charCodeAt(offset),
0,
);
}
offset += 1;
- this.excess += 1;
- this.runConsumed += 1;
+ excess += 1;
+ runConsumed = 1;
}
// Check remaining characters in the run (packed two per uint16 word).
- while (this.runConsumed < runLength) {
- if (offset >= inputLength) return -1;
+ while (runConsumed < runLength) {
+ if (offset >= inputLength) {
+ this.treeIndex = treeIndex;
+ this.excess = excess;
+ this.consumed = consumed;
+ this.runConsumed = runConsumed;
+ return -1;
+ }
- const charIndexInPacked = this.runConsumed - 1;
+ const charIndexInPacked = runConsumed - 1;
const packedWord =
- decodeTree[
- this.treeIndex + 1 + (charIndexInPacked >> 1)
- ];
+ decodeTree[treeIndex + 1 + (charIndexInPacked >> 1)];
const expectedChar =
- ((charIndexInPacked & 1) === 0
- ? packedWord
- : packedWord >> 8) & 0xff;
+ (packedWord >>
+ ((charIndexInPacked & 1) === 0 ? 0 : 8)) &
+ 0xff;
if (input.charCodeAt(offset) !== expectedChar) {
this.runConsumed = 0;
- return this.emitLegacyOrReject(
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
input.charCodeAt(offset),
0,
);
}
offset += 1;
- this.excess += 1;
- this.runConsumed += 1;
+ excess += 1;
+ runConsumed += 1;
}
this.runConsumed = 0;
- this.treeIndex += 1 + (runLength >> 1);
- current = decodeTree[this.treeIndex];
- valueLength = current >>> 14;
-
- // Record legacy match at end of compact run (FLAG13 clear = semicolon optional).
- if (
- valueLength !== 0 &&
- this.decodeMode !== DecodingMode.Strict &&
- (current & BinTrieFlags.FLAG13) === 0
- ) {
- this.result = this.treeIndex;
- /*
- * The `excess` counter started at 1 to count the leading
- * `&`, which is already in `consumed`; subtract it so the
- * run's characters are not counted twice.
- */
- this.consumed += this.excess - 1;
- this.excess = 1;
- }
+ treeIndex += 1 + (runLength >> 1);
+ current = decodeTree[treeIndex];
+ // Loop top handles the landed-on node (record/emit/branch).
+ continue;
}
- if (offset >= inputLength) break;
-
+ // The number of bytes of the value, including the current byte.
+ const valueLength = current >>> 14;
const char = input.charCodeAt(offset);
- /*
- * Implicit semicolon handling: if the current node has a value and the
- * input character is `;`, emit immediately. This covers both strict
- * entities (FLAG13 set) and legacy entities (FLAG13 clear) — neither
- * stores an explicit `;` branch in the trie.
- */
- if (char === CharCodes.SEMI && valueLength !== 0) {
- return this.emitNamedEntityData(
- this.treeIndex,
- valueLength,
- this.consumed + this.excess,
- );
- }
+ if (valueLength !== 0) {
+ // Record a legacy match (FLAG13 clear = semicolon optional).
+ if (!isStrict && (current & BinTrieFlags.FLAG13) === 0) {
+ this.result = treeIndex;
+ consumed += excess - 1;
+ excess = 1;
+ }
- /*
- * `valueLength === 1` packs the codepoint into the header word's
- * low 14 bits, where branch metadata also lives. Skip the branch
- * lookup on leaves so those value bits aren't reinterpreted as
- * branch offsets.
- */
- if (valueLength === 1) {
- return this.emitLegacyOrReject(char, valueLength);
+ /*
+ * Implicit semicolon handling: emit immediately. Covers both
+ * strict (FLAG13 set) and legacy entities — neither stores
+ * an explicit `;` branch in the trie.
+ */
+ if (char === CharCodes.SEMI) {
+ return this.emitNamedEntityData(
+ treeIndex,
+ valueLength,
+ consumed + excess,
+ );
+ }
+
+ /*
+ * `valueLength === 1` packs the codepoint into the header
+ * word's low 14 bits, where branch metadata also lives. Skip
+ * the branch lookup on leaves so those value bits aren't
+ * reinterpreted as branch offsets.
+ */
+ if (valueLength === 1) {
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
+ char,
+ valueLength,
+ );
+ }
}
- this.treeIndex = determineBranch(
+ // Value-bearing or dictionary node: dispatch through determineBranch.
+ const next = determineBranch(
decodeTree,
current,
- this.treeIndex + (valueLength || 1),
+ treeIndex + (valueLength || 1),
char,
);
- if (this.treeIndex < 0) {
- return this.emitLegacyOrReject(char, valueLength);
+ if (next < 0) {
+ return this.flushAndEmitLegacyOrReject(
+ consumed,
+ excess,
+ char,
+ valueLength,
+ );
}
- current = decodeTree[this.treeIndex];
- valueLength = current >>> 14;
-
- /*
- * Record non-terminated (legacy) match for later emission.
- * (`;` is always caught by the pre-navigation check above.)
- */
- if (
- valueLength !== 0 &&
- this.decodeMode !== DecodingMode.Strict &&
- (current & BinTrieFlags.FLAG13) === 0
- ) {
- this.result = this.treeIndex;
- this.consumed += this.excess;
- this.excess = 0;
- }
- // Increment offset & excess for next iteration.
+ treeIndex = next;
+ current = decodeTree[treeIndex];
offset += 1;
- this.excess += 1;
+ excess += 1;
}
+ /*
+ * Chunk exhausted. Record a legacy match we may be sitting on, so a
+ * subsequent `end()` emits it, then persist the walk state.
+ */
+ if (
+ !isStrict &&
+ current >>> 14 !== 0 &&
+ (current & BinTrieFlags.FLAG13) === 0
+ ) {
+ this.result = treeIndex;
+ consumed += excess - 1;
+ excess = 1;
+ }
+ this.treeIndex = treeIndex;
+ this.excess = excess;
+ this.consumed = consumed;
return -1;
}
@@ -541,9 +717,12 @@ export class EntityDecoder {
/**
* Determines the branch of the current node that is taken given the current
* character. This function is used to traverse the trie.
+ *
+ * See `BinTrieFlags` for the branch-data layouts handled here.
* @param decodeTree The trie.
- * @param current The current node.
- * @param nodeIndex Index immediately after the current node header.
+ * @param current The current node's header word.
+ * @param nodeIndex Index of the node's first branch-data word (the header
+ * plus any value words have been skipped by the caller).
* @param char The current character.
* @returns The index of the next node, or -1 if no branch is taken.
*/
@@ -567,40 +746,39 @@ export function determineBranch(
* Jump table: branchCount consecutive slots starting at jumpOffset.
* Unsigned comparison handles both < 0 and >= branchCount in one check.
*/
- const value = char - jumpOffset;
- if (value >>> 0 >= branchCount) return -1;
- const stored = decodeTree[nodeIndex + value];
- // 0 = empty slot (no branch); otherwise relative offset + 1.
- return stored === 0 ? -1 : (nodeIndex + value + stored - 1) & 0xff_ff;
+ const slot = char - jumpOffset;
+ if (slot >>> 0 >= branchCount) return -1;
+ const stored = decodeTree[nodeIndex + slot];
+ /*
+ * 0 = empty slot (no branch); otherwise the child's offset from the
+ * end of the branch array, +1 (end-relative pointers compress
+ * better). `& 0xff_ff` mirrors the encoder's uint16 wrap for
+ * backreferences to already-encoded nodes.
+ */
+ return stored === 0
+ ? -1
+ : (nodeIndex + branchCount + stored - 1) & 0xff_ff;
}
- // Case 2: Packed dictionary (binary search on sorted keys).
- if (branchCount === 0) return -1;
- const packedKeySlots = (branchCount + 1) >> 1;
-
/*
- * Treat packed keys as a virtual sorted array of length `branchCount`.
- * Key(i) = low byte for even i, high byte for odd i in slot i>>1.
+ * Case 2: Packed dictionary. Linear scan — over 90% of dict nodes have
+ * <= 4 branches in the HTML trie, where the constant-factor savings
+ * dominate over binary search's asymptotic edge.
*/
- let lo = 0;
- let hi = branchCount - 1;
-
- while (lo <= hi) {
- const mid = (lo + hi) >>> 1;
- const slot = mid >> 1;
- const packed = decodeTree[nodeIndex + slot];
- const midKey = (packed >> ((mid & 1) << 3)) & 0xff;
-
- if (midKey < char) {
- lo = mid + 1;
- } else if (midKey > char) {
- hi = mid - 1;
- } else {
- const pointerIndex = nodeIndex + packedKeySlots + mid;
- return (pointerIndex + decodeTree[pointerIndex]) & 0xff_ff;
+ if (branchCount === 0) return -1;
+ const packedKeySlots = (branchCount + 1) >> 1;
+ const branchEnd = nodeIndex + packedKeySlots + branchCount;
+ for (let index = 0; index < branchCount; index++) {
+ const packed = decodeTree[nodeIndex + (index >> 1)];
+ const key = (packed >> ((index & 1) << 3)) & 0xff;
+ if (key === char) {
+ const pointerIndex = nodeIndex + packedKeySlots + index;
+ // Pointers are relative to the end of the branch data.
+ return (branchEnd + decodeTree[pointerIndex]) & 0xff_ff;
}
+ // Keys are sorted; if we've passed `char`, no match is possible.
+ if (key > char) return -1;
}
-
return -1;
}
@@ -634,10 +812,13 @@ function readTrieValue(
/**
* Parse a numeric entity (`&#DDD;` or `&#xHHH;`).
*
- * Encodes the result as `(consumed << 21) | codepoint`. The 21-bit codepoint
- * field fits any valid Unicode value (max 0x10FFFF), and packing both into
- * one integer avoids the file-scope `_numericCp` tuple-by-globals pattern.
- * Returns 0 when no digits were found.
+ * Encodes the result as `(consumed << 21) | codepoint` (see
+ * `NumericPacking`; overlong entities spill their length into
+ * `longNumericConsumed`). Returns 0 when no digits were found.
+ *
+ * This is the sync counterpart of the streaming
+ * `EntityDecoder#stateNumericDecimal` / `#stateNumericHex`; digit and
+ * clamping behavior must stay in sync between the two.
* @param input The input string.
* @param numberStart Index of the `#` character.
* @param inputLength Cached `input.length`.
@@ -648,31 +829,38 @@ function parseNumericEntity(
inputLength: number,
): number {
let offset = numberStart + 1; // Skip "#"
- let base = 10;
-
- if (offset < inputLength) {
- const first = input.charCodeAt(offset);
- if (first === CharCodes.LOWER_X || first === CharCodes.UPPER_X) {
- base = 16;
- offset += 1;
- }
- }
-
let cp = 0;
let digits = 0;
- while (offset < inputLength) {
- const char = input.charCodeAt(offset);
- if (isNumber(char)) {
- cp = cp * base + (char - CharCodes.ZERO);
- } else if (base === 16 && isHexadecimalCharacter(char)) {
- cp = cp * 16 + ((char | TO_LOWER_BIT) - CharCodes.LOWER_A + 10);
- } else {
- break;
- }
-
- digits += 1;
+ /*
+ * Separate decimal and hexadecimal loops: each multiplies by a constant
+ * and runs a single digit test, instead of a per-character base check.
+ */
+ if (
+ offset < inputLength &&
+ (input.charCodeAt(offset) | TO_LOWER_BIT) === CharCodes.LOWER_X
+ ) {
offset += 1;
+ while (offset < inputLength) {
+ const char = input.charCodeAt(offset);
+ if (isNumber(char)) {
+ cp = cp * 16 + (char - CharCodes.ZERO);
+ } else if (isHexadecimalCharacter(char)) {
+ cp = cp * 16 + ((char | TO_LOWER_BIT) - CharCodes.LOWER_A + 10);
+ } else {
+ break;
+ }
+ digits += 1;
+ offset += 1;
+ }
+ } else {
+ while (offset < inputLength) {
+ const digit = input.charCodeAt(offset) - CharCodes.ZERO;
+ if (digit >>> 0 > 9) break;
+ cp = cp * 10 + digit;
+ digits += 1;
+ offset += 1;
+ }
}
if (digits === 0) return 0;
@@ -684,22 +872,31 @@ function parseNumericEntity(
/*
* Pack `consumed` (length) and `cp` (Unicode code point) into one
- * non-negative integer to avoid a tuple allocation or a module-scope
- * global. Callers extract via `packed >>> 21` and `packed & 0x1f_ff_ff`.
+ * integer to avoid a tuple allocation on the hot path. Callers extract
+ * the two fields via `NumericPacking`.
*
* `cp` is clamped to 0x110000 (1 past the Unicode max) before packing
* so it stays inside the 21-bit field. Without the clamp, an overflow
* like `�` would silently truncate to a valid-looking
- * codepoint instead of mapping to U+FFFD via `replaceCodePoint`.
+ * codepoint instead of mapping to U+FFFD via `replaceCodePoint`. The
+ * streaming parser applies the same clamp per digit (see
+ * `stateNumericDecimal` / `stateNumericHex`).
*
- * `consumed` fits 11 bits, which covers any practical entity body.
- * Pathologically long digit runs (>2047 chars) overflow the field,
- * but those inputs already produce U+FFFD for the entity value, so
- * the only observable difference is a slightly wrong advance — never
- * an incorrect emitted character.
+ * `consumed` fits its 11-bit field for any entity shorter than 2047
+ * characters, which covers any practical body. Longer digit runs would
+ * wrap the field (mis-consuming input and leaking digits into the
+ * output), so they are routed through the `longNumericConsumed` side
+ * channel instead: the consumed field is pinned at CONSUMED_OVERFLOW
+ * and callers recover the true length from the module-level slot.
*/
if (cp > 0x10_ff_ff) cp = 0x11_00_00;
- return ((offset - numberStart) << 21) | cp;
+ let consumed = offset - numberStart;
+ if (consumed >= CONSUMED_OVERFLOW) {
+ // eslint-disable-next-line unicorn/no-top-level-assignment-in-function -- deliberate side channel, see `longNumericConsumed`
+ longNumericConsumed = consumed;
+ consumed = CONSUMED_OVERFLOW;
+ }
+ return (consumed << CONSUMED_SHIFT) | cp;
}
/**
@@ -729,6 +926,15 @@ function decodeWithTrie(
let chunkStart = 0;
let result = "";
+ /*
+ * Root navigation fields, hoisted out of the per-entity loop. The HTML
+ * root is a multi-branch jump-table covering [A-Za-z]; see the inline
+ * first-iteration comment below.
+ */
+ const root = decodeTree[0];
+ const rootJumpOffset = root & BinTrieFlags.JUMP_TABLE;
+ const rootBranchCount = (root & BinTrieFlags.BRANCH_LENGTH) >> 7;
+
do {
const entityStart = offset + 1;
@@ -738,7 +944,11 @@ function decodeWithTrie(
let value: string;
if (firstChar === CharCodes.NUM) {
const packed = parseNumericEntity(input, entityStart, inputLength);
- consumed = packed >>> 21;
+ consumed = packed >>> CONSUMED_SHIFT;
+ // Overlong entity: the true length is in the side channel.
+ if (consumed === CONSUMED_OVERFLOW) {
+ consumed = longNumericConsumed;
+ }
// In strict mode, require semicolon termination.
if (
isStrict &&
@@ -748,37 +958,106 @@ function decodeWithTrie(
consumed = 0;
}
value =
- consumed > 0
- ? String.fromCodePoint(
- replaceCodePoint(packed & 0x1f_ff_ff),
- )
- : "";
+ consumed === 0
+ ? ""
+ : codePointToString(packed & CODE_POINT_MASK);
} else if (isAlpha(firstChar)) {
consumed = 0;
value = "";
- let nodeIndex = 0;
- let current = decodeTree[nodeIndex];
+ /*
+ * Inline the first trie iteration: from the root, navigate by
+ * `firstChar` to its child. The HTML root is a multi-branch
+ * jump-table covering [A-Za-z], and `firstChar` is already in
+ * hand from the outer loop, so we can skip the redundant
+ * top-of-loop work for iteration 1 (compact-run check,
+ * valueLength compute, charCodeAt, value handling,
+ * determineBranch dispatch) and start the trie loop on the
+ * second character.
+ *
+ * If the tree's root is shaped differently (no jump-table), the
+ * `rootSlotIndex >>> 0 < rootBranchCount` test will fail for
+ * any input and the loop short-circuits via `current = 0`. The
+ * three call sites here all pass `htmlDecodeTree`; if a future
+ * caller passes a non-jump-table tree they should reuse the
+ * original loop-from-root structure.
+ */
+ const rootSlotIndex = firstChar - rootJumpOffset;
+ let nodeIndex: number;
+ if (rootSlotIndex >>> 0 < rootBranchCount) {
+ const stored = decodeTree[1 + rootSlotIndex];
+ nodeIndex =
+ stored === 0 ? -1 : (rootBranchCount + stored) & 0xff_ff;
+ } else {
+ nodeIndex = -1;
+ }
/*
- * Best legacy match found so far. We store the node
- * coordinates and defer readTrieValue() to the end,
- * avoiding repeated String.fromCharCode allocations.
+ * Best legacy (no-semicolon) match so far, as trie coordinates.
+ * Deferring `readTrieValue` to the end avoids allocating a
+ * string for matches that longer matches supersede.
*/
let bestNodeIndex = 0;
let bestValueLength = 0;
+ let current = nodeIndex < 0 ? 0 : decodeTree[nodeIndex];
+ let index = entityStart + 1;
- let index = entityStart;
-
- // Label for breaking out of the main loop from inside the compact run inner loop.
+ /*
+ * Walk the trie from the root child. The `trie` label lets the
+ * inner descent and compact-run loops abandon the entity (and
+ * fall through to the legacy/reject handling) directly.
+ */
trie: while (index < inputLength) {
- // The number of bytes of the value, including the current byte.
- const valueLength = current >>> 14;
+ /*
+ * Descend through pure jump-table nodes (no value, no run)
+ * inline: 81% of trie nodes are value-less jump-tables, so
+ * this avoids a `determineBranch` call per level for the
+ * dominant node shape. A branch miss rejects the entity the
+ * same way a negative `determineBranch` result does, and
+ * the recorded legacy match (if any) is handled post-loop.
+ */
+ while (
+ // Value-less, non-run node with a nonzero jump offset.
+ (current &
+ (BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)) ===
+ 0 &&
+ (current & BinTrieFlags.JUMP_TABLE) !== 0
+ ) {
+ const jumpOffset = current & BinTrieFlags.JUMP_TABLE;
+ const branchCount =
+ (current & BinTrieFlags.BRANCH_LENGTH) >> 7;
+ if (branchCount === 0) {
+ // Single branch encoded inline in the jump offset bits.
+ if (input.charCodeAt(index) !== jumpOffset) break trie;
+ nodeIndex += 1;
+ } else {
+ const slot = input.charCodeAt(index) - jumpOffset;
+ if (slot >>> 0 >= branchCount) break trie;
+ const stored = decodeTree[nodeIndex + 1 + slot];
+ if (stored === 0) break trie;
+ // End-relative: branch data ends at nodeIndex+1+branchCount.
+ nodeIndex =
+ (nodeIndex + branchCount + stored) & 0xff_ff;
+ }
+ current = decodeTree[nodeIndex];
+ index += 1;
+ /*
+ * `charCodeAt` past the end returns NaN, which would
+ * alias to slot 0 after `>>> 0` — bail out explicitly.
+ */
+ if (index >= inputLength) break trie;
+ }
- // Handle compact runs — inline to avoid 5-argument function call overhead.
+ /*
+ * Handle compact runs first — a single mask collapses
+ * the (valueLength == 0, FLAG13 set) test into one compare.
+ * The run path skips the valueLength compute entirely; the
+ * inner loop is inlined to avoid 5-argument call overhead.
+ */
if (
- valueLength === 0 &&
- (current & BinTrieFlags.FLAG13) !== 0
+ (current &
+ (BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)) ===
+ BinTrieFlags.FLAG13
) {
const runLength =
(current & BinTrieFlags.BRANCH_LENGTH) >> 7;
@@ -834,6 +1113,8 @@ function decodeWithTrie(
continue;
}
+ // The number of bytes of the value, including the current byte.
+ const valueLength = current >>> 14;
const char = input.charCodeAt(index);
/*
@@ -845,11 +1126,27 @@ function decodeWithTrie(
// If char is `;`, emit immediately.
if (char === CharCodes.SEMI) {
consumed = index - entityStart + 1;
- value = readTrieValue(
- decodeTree,
- nodeIndex,
- valueLength,
- );
+ /*
+ * Inline-leaf fast path: valueLength 1 means
+ * the header word carries the entire value in its
+ * low 13 bits. That covers most named entities
+ * (e.g., & < é), so skip the
+ * readTrieValue call.
+ */
+ value =
+ valueLength === 1
+ ? String.fromCharCode(
+ current &
+ ~(
+ BinTrieFlags.VALUE_LENGTH |
+ BinTrieFlags.FLAG13
+ ),
+ )
+ : readTrieValue(
+ decodeTree,
+ nodeIndex,
+ valueLength,
+ );
// eslint-disable-next-line unicorn/no-break-in-nested-loop
break;
}
@@ -944,7 +1241,16 @@ function decodeWithTrie(
result += value;
offset = chunkStart = entityStart + consumed;
}
- } while ((offset = input.indexOf("&", offset)) >= 0);
+
+ /*
+ * Adjacent entities (`&x;&y;`) are common in entity-dense input;
+ * checking the single character at `offset` first skips the
+ * `indexOf` call (and its per-call overhead) for that case.
+ */
+ if (input.charCodeAt(offset) !== CharCodes.AMP) {
+ offset = input.indexOf("&", offset);
+ }
+ } while (offset >= 0);
return result + input.slice(chunkStart);
}
@@ -1014,7 +1320,11 @@ export function decodeXML(xmlString: string): string {
start,
xmlString.length,
);
- consumed = packed >>> 21;
+ consumed = packed >>> CONSUMED_SHIFT;
+ // Overlong entity: the true length is in the side channel.
+ if (consumed === CONSUMED_OVERFLOW) {
+ consumed = longNumericConsumed;
+ }
// XML is always strict — require semicolon.
if (
consumed === 0 ||
@@ -1022,9 +1332,7 @@ export function decodeXML(xmlString: string): string {
) {
consumed = 0;
} else {
- value = String.fromCodePoint(
- replaceCodePoint(packed & 0x1f_ff_ff),
- );
+ value = codePointToString(packed & CODE_POINT_MASK);
}
} else {
const c2 = xmlString.charCodeAt(start + 1);
@@ -1078,8 +1386,16 @@ export function decodeXML(xmlString: string): string {
result += "&";
lastIndex = start;
}
- offset = lastIndex;
- } while ((offset = xmlString.indexOf("&", offset)) >= 0);
+ /*
+ * Adjacent entities (`&x;&y;`) are common in entity-dense input;
+ * checking the single character at `lastIndex` first skips the
+ * `indexOf` call (and its per-call overhead) for that case.
+ */
+ offset =
+ xmlString.charCodeAt(lastIndex) === CharCodes.AMP
+ ? lastIndex
+ : xmlString.indexOf("&", lastIndex);
+ } while (offset >= 0);
return result + xmlString.slice(lastIndex);
}
diff --git a/src/generated/decode-data-html.ts b/src/generated/decode-data-html.ts
index 56beb2aa..ef4d993b 100644
--- a/src/generated/decode-data-html.ts
+++ b/src/generated/decode-data-html.ts
@@ -3,7 +3,10 @@
import { decodeTrieDict } from "../internal/decode-shared.js";
/** Packed HTML decode trie data. */
export const htmlDecodeTree: Uint16Array = /* #__PURE__ */ decodeTrieDict(
- "%}1s%}%%&}%%}#%%~!60%/~!J~!J~%L~y<'*)~!A,~~#y=~'J~01~-6+~*V~*<1&~~#G3~#p~~#cA~7z!#5}R%}:&)%#)(%%#'}(%#%}(%}#%#%%}&%&##%#%}&%#&#)'%%%#%%%#%}#%#%%%}#%},%%}#%%#'#%)%##&#&'%}#%#)&&(##&*)#%#&9##%#%&#(&%,*)&(%#%%&-%8&%,&/#'(%',-'3)'&O5)-0(#+;*A,-#09~!3~!Je~#g~!Q~#T~!0@8WT~#|~(A~'V~&+~%P~#1~#s~!y~#_NT~'P~%8~#(~#S}#%}*&}#%##%##%##&#-}&'#'&%#.++}%mI,#,@&(}*%}*'%&##&#%##%}&0}#.},U},%}+%}&%}#%##&}B%(##%}#%%}%%##%)})%##%#&}&%##%&}<%}>%#%&}*%}(%}9%}/%})%}*%}*%}?&}&%}3%}&*#%})%#%#)}#&#%+#+*%E%%'%#%##%}#*W#&##I}#&&##%&%##%#%QX-&%%))@Y/0'&#&%#(#.%-''''++++7}>%%2'',##1,#%#&%##&#'##&#*#9)%&%}#*}%,#+U%A&%#'&}#)d/SD',9E00#y#@}(+}&%&~!:(~!X}#*}(&&}(&}(,%}%&#+&}#&}I%#%}%)#(},'%#*}4%%#%}(''}#/##(##),%-##%%)#&}(.}&%#&}%%}*&#%},&&}&%}#%*'#%})%}D&}4&}6&#&}-,%}#%})-~#'~#`Z~!4[~#`~,Z~#p~#q~#q#~*O~8?'9%~!,#%})%})%}@%}?%}(~!?~#<~#pP'~#>##E~#=1#%K+~#?#~#pJ)~#>&#~#lF1~#A#&~'X%&#~#l#Q~#N~#f~#j/~#f~#m#)'~#i'&)6%(%~#B%##%&&#%%#~#_%#0%~#]4}#32~,w~2+#:&#%&'0%&>%}#>##F+)#%&&#(+_}4&}-%}(&}h%})7Fdf0@+/v4}&WS%##&/1&'('B#%}.%}'+#%}#%%&#&%#%##+#&#)#6#'#.},%}c%},%#%##%&#&%#&~#>'*-.}%%##%}#%%}%'~#)D/}&%*&~#_}&&(~#S2##&/}#~#=##*'4%}&')}#%&~#^#~#h%%,#&~#B.5}'%##'%#~#O1}%%}'+'~#8*;%##%%#%#%(&~#;#@%}#&%}%(#(~#H1}'%&}#&&~#?A}&'&#~#@#%31}(&##%#&#~#[}'%&#%}'%~#9C##%}%&}(%~#=&%,3}%'&%}'~#]'#&&)}#'~#Y%-'}#~#]-%'%%~#Y%%&#&&&}#~#b/'~2d*&'~!W~&9~#p~sg3}%*''0})&}+~!8}#-##uD*)1~!J~!J~!J~!J~!J~!J~!J~#p~~#LH:5~%T>~#xU:~#j&~#':5~!v1~!G~!B&&~!=|%#&&#&}#%},%%}'%}+X#%}#&}(%}'%}<%}#%}%%'}'%}:>TT*~!-.4~!(~#)~!f64A%-%##{>-*ehXMA~!4%kj4s,V0'&0dH*%%*##'%7-K.&%-%)#(#K~!176P,0B*N<#;e44%%6&.)-,),%KV,#%.(%+'3++#('#&%#%&*%&)F#ZI*9*'%E@/2<+)u)0&+E/#GA6,91?0#5H7&.`-i#/YE''-1?503)l,~!8/#%4I{~!5=).;9~!+~!%)A~!R)(0#pws+u=uxv=v^w/wW13a%a(a+a2a5a8a:aaHaOaQa[a_OuJoxaboQglwE7osRwY7qfIwt!%@5ott%@`_y=qh8`_ylk+wuwGs?uev>]e2fcuKowIw#!&@`_xofcv9]gaQtz7ou<;ovd,Tu?ZwV)a,a-a?aBaFaHaJIvp!(ifv!v~as]g&cE#%]o776]iGEt&JvpB#(-gjw27]gBj!wRu|w)]eU4sB@`_y=?8`_ymNu7s/5G~*guu2av]gZd:pzTu)uPWZwV/13a>aRaTa`aaabaub/bUbWbZ5Etqpf_Woy@5qlgvv:B]g=>BqnIv7!&Qu|4qaimvrvBBq`G}|:sURut7Kpvq6%(,/>B]hCQwQA]h@Ow2]h?Qu0A]hA>Iw#!/n)ugu4vAw?wRpnw?uHrv;]g?idwFu~EIp}!*jgs-u/wIw@7]dwg|w@7]dtKv=x6gD]n1Ju`B#(+i:u6w?]gj=B]geaiqrt);xCfw4sC@`_y>Iu;!a=Iv'!a5j&wVKpYq2%(2a&glwE7q_>cD##s0j+u/s6wE7s4RwX7qUQtz7s3fgt{]hkj_vrv8ueVaV]ebKwAwh%(a%b78`_ynxMb?#'>B]e8gyS;]g]f]u/ctp[q%qB'a&a1aj9D>=B>C@aZ=B79@4bk}W>Iwg!!qZihvkXD]fRIvF!a*8BJp~af#(.RXD]fOj7UpT[we]fS7F}a=9Iq%!a%Nw9Ip~!'RXD]kYj7UpT[we]k[j7UpT[we]kZQUBIq.!'RXD]fQ77]hP?IpY!'RXD]fPj/v@vkXD]fTj^wCs>VrE@]g2=ctpUq%rW'a)a,aKaaaiRXD^etq9!&4@]klijvkXD]flRwY7s5Nw9Jq7ah#+1jbUqDs=vN@]l1j,qCs=vN@]l?i-w7Zx+f<4@]l7QUBIqA!)j,qCs=vN@]l@i-w7Zx+f@4@]l877x*hMRXD]f+@@>bt}ZIw7!&@`_xqPuoqtd>q5Tu>uxvKvjwDwn123a'a3a5a8a;a?a]agajatABawaXrMaYp&glwE7o{JuZE#&(Pv>q|@5o|tB>Bqx@`_y?RwX7ozi-u0w?]fxIvR!%5@qvBEIq@!*jfVq,wTY7]k0kIwwv#Vq,wTY7]jqIvX!%>=qz8`_yoi;Wv>sDCIuZ!*;x1n2Qtz7]gNi~s,ucv1]fKIu[!%@]eX<]n04sF<;o}IvY!&fjw1]ftj:u1w?rmpb]eccju=vHA&'*a3aFEt9@`_y@gtu/6Iq@!*jfVq,wTY7]k1kIwwv#Vq,wTY7]jpJvzC#'*8`_ypf@u|]fqj&u-wCuC]eY5G~/d8q3s+tvuIZwD-//3a'a3a5a8a9a=ao45EsuoSferqx5cysmRwY7r#Ju^E#&(f_Wr(@5q~t(>Br&@`_yA]i#?8`_yqi-w5vrctpeptq8'/a&a+a.a4gyS;x/gnNw0]i&j0u|vaS;]gpikriu5@]n]Nw0]g~j8v5pcwTV]n:Qtz7]gz5@`_xr]gsd,rDu=vHwV),3a&a(a/a=aEg_s8Et?Iw7!%7aus.qSfcs@r)@]e>jeu'wCvQT7]e=Ivz!%8]e?k'vPw?VuXu1]j=Iw7!#G~'Puor+D=aYCBr3@]eCRwX7o~^eCvR!.IuF!#@r/ierrwwaZ]ed;:7bq}?Iw`!2x6g9Ivt!&PV]g8j's=uev>]hiieudus7Iq/!'gwv04]e4Qu0A]e3JvXB#&)>=r18`_yr4sH5@]eBQtz7r-Iv.!%5Esx;p%cju=w,C&-0a%a/Iwt!%@5r7t.@`_yB?8`_ysIu(!&@`_xsPwqszfdwqsvcxpqTvGA(*,.a&a)a-5Et:5Es~fhrtsIIwr!%f_Wr9t/@`_yC?8`_yt5@`_xtd4q3Tu?v/w,B,..a9aEc/c0c3c4cIcN5Es{oRcjv'v]@&(*,1flu5rr@f_Wr>t0Iw'!bOBd2pVpdq7qBY+a9a>a_ac4b&b=}6bAIvy!*j`u/vlTu.B]kPPwe^eqq!!&4@]f_j7UpT[we]fEidWv:9]iH>Iwh!*jZu/vlTu.B]kN=IqA!)j,qCs=vN@]lBi-w7Zx+fB4@]l:gtvM@]iJQUBIq>!'RXD]eui-w7Z]l/Ivr!27^hLq>!'RXD]f(i-w7Z]l;iev5ux7^hYp`!&4@]m7gyS;]h[?Jq0ah#*0j:v@u%w7Z]l2j,qCs=vN@]lAi-w7Zx+f>4@]l9i-w7Zx+f;4@]l3:9asB4@@>bt}3Actpeptq8'.a%a*a-a3k,Spgu3w5vr]i%j0u|vaS;]goikriu5@]g}Nw0]n[j8v5pcwTV]n9Qtz7]R@`_yDx6i!i{rx[we]fYQvEBrBJv]D#aOaR9Kq%vw%a'a-abt}0:9asB4@@>bt}/8`_yu7@Iq%!)j,w9vkXD]ezj7UpT[we]eyJuOB#%&G}N]f2PuorD]grd,Tu?vHwV)+,a-a0a5a9a;?]kdEt1Iuu!)j.v1vQT7]e0igv:v|8]eZ@`_yEj!w2urw2]g#?8`_yv5G~)sKd0rMu(vGwDC*,/a,7ascbcccd5Es|glwE7rFJu&E#&(Pv>rJf_WrHt2Jw(D#a:aUi*ueu7Jq4ah#+a(j^u]v1vQT7]dias:Iv6!!auae?45F~+7@Eafas:bm~,fktwIpt!0@74B7@aX@74B7G}97AAa^7Abq}agXv:7qG@`_yFKv3wA%)02Rriau]e1kBu3ukv:q+rtu(qY8]eFxYo;p[pfpup|q)qA-a,a6aO?a}bNb[blc!c?JIwR!(j!vtu6w?]gkgxrF?]guk_s-u/u%wCs>VrE@]g3Jvhax#(1i-u0w?]fyflVx1giQtz7`gNLQwDA]fuikriu5@xTgwpeptq8'+2a)a,a3gyS;]gyj0u|vaS;`gpLikriu5@`gsLNw0]h#j8v5pcwTV`n:LQtz7]g|flv[c6#*j#v@wHv[`gZLgyS;`g[L7Iw'!a'j;vnrmuKu/^i2p`!'4@`m7LgyS;]i4AxSgvpfq(af&*/3a+gyS;]gxikriu5@]h!Nw0`grLj8v5pcwTV`n9LQtz7]g{i-wDtwIpt!,kCriu5phu3w5vr`n]Lizw0u#w0`n[Lj&s=tww&^h+q&!(gyS;`njLj8v5pcwTV]i)Iu^!+k@vru4uqv)v8B]f{k&w;vnrmuKu/^i3p`!'4@`m8LgyS;]i5IwT!a0inYq*CIvS!,fjw8zHh9LgyS;]i+i-w/w8zHh:LgyS;]i,JsVrE@]g15@`_xuQtz7p'sLd:rHtvuIvKvzwDwa/1a%a,a/a2a5aDaHa[a]agaq4OuJrUglwE7p)Iwt!%@5p*t3gkrp5rS@`_yGRwX7p(Ju&:#&(5@rO94sWglX=sN?8`_ywizpWv}wuIp}!*jgs-u/wIw@7]dvg|w@7]ds]myIut!&@`_xvMuRp-:cB#&67p+7A]mc<;p,7@Ip{!a&IY!%@]e*45Ium!#]iw7B]irjYu3w?u,udA]iud0TuPWZA*/03a%a%a)a:apikueVaV]fsEt4@`_yH:sTsOj(pxv:w2q^IvY!-=54@7?;4=F}'8]eJxQntu^>#a/a2i,tww&xQh%q&af#(.gyS;]njj8v5pcwTV]h'Qtz7]h)<7]e%IvU!&flw7]f|iiwCvH=x2gD;]g*Iu[!&@`_xwsVKu:w,%'*-aaafoP@`_yI?8]eK5@`_xxd8p`Tu?uaZwV-01a3a?a`abacbtb{b|c%M[]kiaXoaJv6@#&(flu5rW9]kS@xCf#;]kmJu&E#&(Pv>r[f_WrYt5xEeMgnw/7Iq;!2Ivh!'i-u0w?]fzjsPfaw;d,pVpdq=rY)a7aIwh!*jZu/vlTu.B]kO=IqA!)j,qCs=vN@]l>i-w7Zx+fA4@]l6gtvM@]iKIvr!27^hKq>!'RXD]f*i-w7Z]laAadafalapb0b3b5Is7!%fGwqt>Et=gRs;EtAglwE7r^xRnuu&wt%')+Pv>rdf_Wrb@5r`t6@`_yJfgwCKpsq<%+1a(j/v@vkXD]etj,w9vkXD]eqj7UpT[we]esijvkXD]erfarqsQj4u|uUs@u/]g(?8`_yxIwU!%B]g)Mu3xQjoq'ag#+a3k)u5w/s=uev>]h=CIvS!+fjw8x.h9gyS;]h;i-w/w8x.h:gyS;]h5@`_xy4@]hmKsa[59;fNpyp3f@pa]ePIs:!%5Es}Et;IwK!!qFsRJu&E#&(Pv>rhf_Wrft7@`_yKIu^!-IwC!'i-vGu3]gA4sGIv6!)ifvQT7`e0dhhzrtu(]dgOu)xQgHpeaf#(.gyS;]gOj0u|vaS;]gQQtz7]gT?8`_yyi}u}p]w@]e7Iw7!&@`_xzPuorjd:s+tvuIv=v^w/wW/a-a6a=a@aCaFaH4aubObPbRbTIvp!%flu5p/@x>f!f^vv]l,@Iu(!#Et!aw7rpIwt!%@5p0t8gkrp5rt@`_yLRwX7p.MvprnIu]!a37@Ip{!a%IY!#@qT45Ium!#]ix7B]isjYu3w?u,udA]iv>=x0hjOw2]h8IvX!%>=rv8`_yzd,pYq1ttw-)a,a2a9|ma@aHaZRXD^erpZ!&4@]kkj/v@vkXD]fDj/v@vkXD]evjhufubvowO<]lO77x*hNRXD]f)>D=4@@>bt|l7@Iq%!)j,w9vkXD]ewj7UpT[we]ex:x;d+>=sSfcuKrr5@`_x{Qtz7rl<;p1d0s*tvu?w,aw*-01a'aZa^abafMuR]hS4@]o:Et'MuRx;hQ]o6Ivr!#]hhJw6E#'a54@]drx:drf^VKprq8%(*04@]g0fcu1qWj,rtrvvN@]kHQtz7]gLj3uaq-rtu(]dh@`_yM?8`_y{5@`_x|gmv~as]hRcju(vGA&(+.2fcs@rxf_u+]hg@`_yN?8`_y|5@`_x}Ku_w,%((,@`_yOsM?8`_y}5@`_x~d0plrXu=w,C*,.03a+a.a2a65EtD5Esy5EtCglwE7p2Iwt!%@5ryt@@`_yP?8`_y~5@`_y!<;r{d,rKtvu?w,)+.a&a(a4a6a95Et+glwE7r|Iwp!%Pv>s#t,>Br~IwC!)>ai:6Bbg|~4sE@]eS?8]eQ5@`_y#d>s+u(uIv/vLvzwDwh1a%a(a=a?aCaFaVabb@bHbUbVbbFbfglwE7p6RwY7qgxSgJtZwOE&()+-`gJcv]gK@5p7B7ogtEOuJp;xAe2`_yQRwX7p5IvV!,IvW!&fjv2]e]bg~0as4sXIvR!|QIut!#@qe9]miIuG!a(xRg4ttw_%()-=6]mz]n#gtv^7]m}]m~xTg-uvvxw|'(*a7a>aE]lq7]g-A6x2g.d/*+,-./01]lu]lv]lw]lx]ly]lz]l{]l|BxEg,arx5he]lpIwA!%as]g/qcM[]ioIvX!%>=qi8`_z!xTgTrHu^v^'(+,./]n-f^vv]n,]gV6]gWAqKPwlx6gTbp~-fcuKp:Jw7E#''@`_y%qL=:=H|X=B]mJd>rQsB]obr|MJuMD##%sY]e^Nv8]gt@`_yR9cxvDwDwaD(a'a9aDaRaVaWJuZC#%(bo|k@5]k*?]hjJvUB#'*>B]mBOw2]mCQu0A]mDIwB!&f^v`]mF4@]k2j&rmuKu/IwM!&fgv@]jz?]jugxwPA]mE7F}=F}>gjXD]kfJuk>#aFaTIv6!a>auJw*B#).iiu8uK7]m?avC4@F|ej&rmuKu/xQjvuu@#'*fgv@]j{Nw9]j}QUB]jxau]j)IoK!*IoL!#]jm]jlaF]jn5au]jkIvF!*zLaOe9flw[`gje9B]iPKwAwo%(-08`_z#xChN><]hNfku-]hod8pjqBtuv+wAwa-a,a;aIa_abapax]m4Kq%vw%&'(]jM]jJ]j@]j?xRj=q:wM%&'(]j^]ja]jE]jFQwQA]hHOw2]hGQu0A]hIKq%vw%&'(]jS]jP]jB]jAxTj>puuLvw'()*+,]jc]jY]jV]jG]jD]jCIwY!%awF|Ef]YoYKu(vH%(+.@`_y&<:]e.B]mj`g6}BIvF!%B]e+bm|`Ku&wO%.02Ivz!%A]mv>=qqf_Wp<@5qm?AxBmu<]mw>BqoJv(=#&*:;ojgxwyaw]m!Bw~b<@6>H|D@`_ySJu(:#%,Et]5auxf^vv]m0flw!xDk8:B]k8Kv/v]%+a6aB>=x6aLxJga|9Iv[!&4xCa>qP^fruw!#bm}%7Iwk!&Nw?]fr7bq|CIu`!(x5gQ>B]n*=H|AJvsE#&(`_z%>bd|@x%b@@]eHIvC!&@@]f6AA]kEIwL!&@`_y'IvS!&x6o(]o*x6o)]o+f_w@]i7cxu)vZw^D(1a+a0aRaqazM[Ivw!#]l&]l!Iw-!%@]i'5]i(M[x?f7]l)xSg7sB]h7@]mn`g7}BKVw^%*a6a8@@x=:=H|8=B]g>gtw7E]i]dApis+tvu?uauzZwDwhaya%a'a*a?aIaPa`an8>b9b;bCb}c,c2c5c8c:@G|?4@]lFKuxw/%(+,favr]dyNuS]e`G|zasxEdn]hLcA#(gjXD]kh4bc|=Iwp!%Pv>qstI^ebvC!)Ivt|;a#@]fIg{u4av]n3Jux<##&ocB4sZgxwyaw]l~Ivv!&fjw;]l_`_yT4@Ivw!#]fB]fAcju&w(aw&|>2a&a)<^hkw,!(=6xBhkflw<]k:gjv04sn:=]i:^btvH!,67x#btihueu0A]hn=bu~/5Etg5IZ!&@=]iW>?]iMcjvZwWD&(+a8aCOYqI8`_z&xRcqv)w-%+/2avx5g]>B]g^QwQA]gEOw2]g%i9Su3]hJar;7ar4@D769F|!=Jttas#(1@@>bt|w>D=4@@>Dbq}]ibv_vM=Ivw!&78H}b:9asH}ac<#)au4@>bt|qIZ!&@=]iX>?]iLJvDB#+-Iww!%`_y(tj;]mAPuoquIvq!&>B]i9:xHj|}2IuM!%@G|<4G|ri*uKu/]lsIu[!#Etsi/rv[]k^d@rGtvu?uxv=v^v{wDwn3a*a4aF|BaGaQa]apaza~b*b>b`bibrbvbzIvB!}_>H}RIw#!%flu5p>fkvr]n+KuZwv%'*-Pv>q}@x4gcp?Ov>]gbtc>BqyIvm!&>B]g_`_yUxPnV#&Mu7p=x5nR>B]nTxQnUWA#()i6u5w/]i|]eEx5nQ>B]nSJvRA#&05@qwBE^fvw_!#7B]fv?x)dbc2#%]dc]ddIw(!!rN?]daIvX!%>=q{8`_z'JVA#),@xBh|;]m=s[sqKw#wa%,a0aBIvH!}e@5]gcIW!#bl|:Mw?Iux!&B@]nRNw0]nQJu&:#&);AqOAB]ghawx-gjaV]n4i8Yv!]m>IrG!&>B]g`@@]lRJtv:#&}w@]eW>H|%IuM!!s^pEIvx!#;p@>]e5Ju[?#%';qHAH|7IvF!*5B4B:>bm|4=7=B:4;F|6d8Tu?ujvBA7bp}TEtYguV7]k5JW@#'1OuJ]},IW!%9]}):9]}-`_yVOuJ]}*OuJ`8atJVB#&)B]k<:9]}+=A]jt>8s&Ivz!&8`_z(Iuk!%;bk|2xEh{]o2ibwCv:B]mHIvC!aUIw#!aNcz)a+a2aa!aDc{))**+a!+oo]eeon]eg]ek]emIoM!#]ef]ehJoNaJ##%op]ei]enaG]ejIoO!#]el]eoaJ]ep;]e-D=]iY5@`_y)d?rHsBr'xQgnvhA|/!,^gnw.|0a#;4=H|1xQn:tv;#&-5]nd>Bx>n]n@zJi&}BA]nP@`_yWxIgs|.feuv]e_5EthxQg~rHat#%&]nN]n`]n_KrHw&%&,0]Q?x?nFPwl]nFx@nDxJnD~,:<]i/?8`_z)4awF{kIu[!%@]e<<^gzuv!#]nJ]nLx'aPtvvh@&.1a%a)Iu[!#]nb@]n6>B]h~fOY]llg|w&B]n8cjttuvA&~)0|,a)Ivz!'?@>bu}^@]lXavIvh|)a#;7Abq}Y:bl|+Iv8!)ikv?vdav`Q}BbO~/d2rDs]ha@`_yXAIwd!'gjXD]kxgjXD]kycjv%v^@&),a/a2@@]fpfkw;]gGauIvw!)j,w9YXD]f,j7Urx[we]f-8`_z*f]Y]dqJutB#'*@`_y*4Abg}mPuor,IvS!&flu|]e,fbv8]dnd;Tu?u`v-vLvjwDC03a,a4a:a=aVaW:b9b6bDbGbSbUglwE7pB^e4wt!%@5pCtMIwj!#EtJ5;oUIvs{Sa#`_yYRwX7pAxQedv:>#.1Iv:!&=B]mGB]g:f`v:]m9B4]eTOuJr6JvC?#a,a.JuFB#%2@r0Juv?{~!&:=F|'4@H{}asr48]h_76s'xRfxu=w@%(/2Mu3]e;:=xCg+:7]m:6>H~!xRg8u(vZ%(1a&4;]hbIvt!&7@bq|pbc~,gjuQau]mPPt|]mhKuFwA%&(+Etf>=r28`_z+4s_g|w&BoqIu[!&@`_y+=xRfxtZw_%&){c]i@>B]i=xEi<]i;x:e3Ou)r.Iv.!%5Etk;pDctu=vKwV'.13a(a2Iwt!%@5r8tN@`_yZMuSs)?8`_z,Iu(!&@`_y,Pwqtmfdwqtid,TuIuiw,)-a%a(a+a-a/a3fhrtxEd!soIwr!%f_Wr:tO@`_y[Ru*=r;5EtZ5Etq?8`_z-5@`_y-dGpUpks+tvu?Uuzv=v^w/wWawa)a7a:a=a@b+bIbQbfcGcKcMcPcQcScVc[cfcgckcncrcsJYB#&'@G|-G{9MW]kpM[]kgx8go]nG4@]lCd0u(v*v]v{B*,03a&a/a0a2aVflu5r=i5wAwb]m%@4bm{zf]rhs`9^kPuu!#]lhF{{]nAC>o_@xUequrAIu]!%:;r?bk{vbb}}tPKvbw/%'+a(4]l#C>xKdv|oIwM!&fbY]lHgzrl@]l.as]f4xRgmuIw.%{|adamBcjuMvwB&-a,a2aRRXDxCeq4bh}fibv_vM=IwM!&fgv@]f>=bq|JavC:94@@>bt}:j6u3w8v,w&]hr^gmw.{xa#;4=H{yxRn9tvw(%'.35]nc>Bx>n;xAn=]n?zJi%}BA]nOcjttuHA&+.a*a,??@>bu}S>B]h}avIvf!%BG{s9BG}OBG{t:bl{uJW@#'*fjw;]l[>>G{m`_y]x.g}]nMc;#,@IwM}ca#x;f;]lK;au]jj5EtnxRgrTw;%',/@G}`>@=7G}!Mt}]lL@:]k/IvH!%f_w@rCflwDx2ipf^u,]ipKrHw&%&,0]gq?x?nEPwl]nEx@nCxJnC~,:<]i.d,s+vLwAw})2a%aJaYaiam5Ivy!%9]kT@]fn@bj{e9Jv/@#3a(Nw9IY{qa#:9asB4@@>bt{qi*w-vN]k]:9asB4@@>bt{pijYXDIvw!&78H}IQUB]f/Ju;;#&(@]l``_z.CA]mYQu0A]m`c;#'AB]g'ba|PxNk!#|jffu+]k!4@x;a:B]ljcjTv+B&(-23@G{T>@=7G||4@x5fJ]lN]dl@:]hfctTuawB'*{n,a&a.fivO]e(@`_y.<^RuH!#]nI]nKIwK}Aa#>xAds]duPuorEx(aNtvuavh@(}P/2a&a)a-Iu[!#]na@]n5@7F}Mfew&]hpM[]lWg|w&B]n7IuY!&4@]lmxNj~{S|h@IwM!'gzrl@]l-fbY]lGIv8!)ikv?vdav`gq}BbO~/d:rGtvu?uavxQf*uuC#'*>Dbmzu78H{Hbo|Ofdvr]jsIwv!&ferq]mWtQMuR]dpk%wVu3rhuKu/]g.@`_y^>]eRJtv=#&a'@>ohxQg0T6#&)AH|a:@]o?>BoiCA^g!tu|ia#xDgE]mXc=#&?]o4G}(?;Cbq{dIvU!&Nw*]hO8`_z/Iw7!&@`_y/fhw,]gJxOd##(ibu{wC@x2k=;xLk={bIwV!#?oT=rKf_WrI=9x5gS>B`n*L?]mktRMuR]doxTgirDvcwm'*a&a*a-a6@@]fV@Ivu!%au]kwx>exbt|F>B`g]LC:bs{RIu^!&4@]k{bl{WfcwDxLfu{X@`_y_Ku!wD{_#2a%^gyw.!{b^gyw.{[a#;4=H{`:bl{axAgw]gwJrD?#&)@G}X@@]f14@]oA^fzw_!|&x5iC]iA5Etocxp_ttw&B(*,/1aKaM@G}U`goL@@]e{@]d|xQgxveA#a(a1BIY!'@@>bt~(:9asB4@@>bt}s^gxw.}{a#;4=H{ZzNn9L]gv:bl{YxAgv:xGi2{Q:bd{jIwA!&8`_z0x&bBv:!a+=xQfytZaw#&*`i@L>B`i=LcY{E#%]i?]i>:xEf{cY{^#%]iE]iDJvC@#a(a,@xQg3v~B#(+;;7bk{B;`oCe9`fsLgtv:B]mM^h+u(!%CF{PzInjLx6h+bp{MKrDw<%'3a*@G}L@@^e|wc!%`k~L`e}Lj-w;YXD]e|@:xGi3{OcxuOv,vjC(a%a.}da2a4a=xQh,u(@#{U%CF{V`_y0fgwCIv[}aa#4@ba}n]eG?]ded0pjttu`vwA*-03a'a0a4aHaTMuR]hUM[]kc?`gYe6MuR]hTIw8!%`gne6`aPe6gvu_=]m;Jt~B#')@@]ka`gme6zMaNe6:7`h[e6Iw4!&@@]kbPu-`h]e6:<`gHe6JrD=#'2@@]fU@Ivu!%au]kvx>ewbt|5NY]kzd@rVtvu?UWvKvzwDwa|_2a)a1aFaHaRa^|badaub#b'b4bNbTb^b_Iw#!|^flu5pHIwt!&@x4hDpItScjs+vHA&(*zD+Abg|]OTrTaw]mdfgtz]m+OuJrVIvp!&:@]m-`_y`JZB#%'=s2Mu7pG]m/Iv&!{C4@]m&KTw<%'13@G|YIvv!%@]m,fgw0]m*=F{F]m.Ju&:#&(5@rP94shJtv=#&'Pv>sb]m'CbqzC?8`_z1Ju&;#&)@]m(@?]m)CbqzBxTg5ttvHw_')a(a+a-a1@G|VxQn%u?<#))@x>e[8]e[o^olfauB]h^@]m{gtv^7]m|]n!Jut>#%'G~%MuRpL;]hB:cB#&67pJ7Ax2hAA]mb<;pKf]Y]i`d8Tu?uav/ZwV-a4a5aLaOa_aj{Ab-b:c&c*@xQg2v~B#){=x!bJ;7bkzEIW!%<]oB]oCEtT@cju[v[B&(*|G,=BqJ>6qM:;]d~Nun]e!@`_yaJv,>#&)xEd*sk<4Hzg=7]k4^d&w`!(icuAZau]h{slIwJ!.=Iul!'aux9eA]e@bs|uAxVa=s+tvv)wD)|K+.3a%a&a)f^vv]mR:@]mQIwR|da#]mT]n/=od:<]mUD>]mVJvYC#(+i6uew?]mN8`_z2=6oVxWh%rHu(v:w,C*+-/aPaTaZaJ5]nl?]npCF{7x4njxSh%Tv8A&+12a5??@>bu~'C@;E7bp{6bp{5Ju&A#(+i8vzwl]nravav]nn:<]i0:bl{4<7xLe#{0JrHA~*!~*bo~%Ju>?{3!a%JVA#'*OY]i^fcu1]iQfluC]iRxCg*bn{1Puv]hXIu[!&@`_y1sggvw#?]dfctu_v^wV'|t)-1a&@`_yb?8`_z3Rv,7]e/5@`_y2Ju&>#3a)BIu^!)@=:>=bqzG=B]mOABx6aQbp|gH{,dCpUrKsom@xXess+u=uyw-wg+-12a%a&a'a)a,a.?]lVx7NA]ku]k~A]ksbj|nbo}#;]l*:<]lU;]f']e}IuZ!&:;]ko>x=gC4;bq{'Js+@#&)@G|3@au]kKIuk!)5Ium!!qXqRIw&!#]lc;IwM!#]le]lgKu&wz%'/0Pv>r]Iu]!%:;rZbk{+bb}}tUKutw.%'+/4]l%gmrl@]lJC>xKdwxmas]f5JT9#2{!;xQeMvYA#&{-=F{/4@H{&B]jrJW@#'*fjw;]l]>>G{)`_ycIvC!-@IwM!#]f@x;f?]lMxEd'spJv9A#adaeasBctuMvwwD'.a-a@aGaPRXDxCes4bh}[ibv_vM=IwM!&>Dbm}z?]f?Nw9IuM!(@@>Dbq{#4@?>>=bqz>jbUrx[weA]fHavC:94@@>bt}Vj6u3w8v,w&]hs9s1:=96>BA7bp|ZJuM<#&(@Gz~4Gz=]dmgww2Bx2iqf^u,]iqfety]o=Ks+wA%.0a0Ivy!%9]kU@]fo@bjz}Ju;;#&(@]la`_z4CA]mZQu0A]maIvR!(@x8a;B]lki7ubw?]mK4@G}lKTvg%({%*fivO]e)@`_y3IwK}Qa#>xKdtxhJua@#'*@7F}gfew&]hq:xQjyu?;z||(a#fkuc]m6i4uTY]lI]eOdAs+tvu?uav/v^v{wDwhaya%a(a+ahapb*b.bFbcbec!c-c0c@cAJcOclcmglwE7r_avCbn|xxWh&rHu(v:w-E*+3a&a*a,a:a>a@]nmIvz!#]nq>=reCF{*x5nk:;rc@5raJrHA#%']no?]ns:<]i1i7ubw?]mL:bl{(tV>B^hlu'|Ia#]n'cxrDv'wDax(+a&a'a(a+a3@@]fW@Ivu!#bj|fx>eybtzFBoZ:qNfmY]k|Iws!%5Et_t^@BIv[!%:bd{>4@4bk}4o`Iv*!*<4^d(wZ!a#scxUgHu)uxv]@(+-03a&a)>B]n(xJgO{(x.nX]nZx.nW]nY7]gROw2]mSM[]lS4@Gz{Ku&w<%3a(a.Iw*!);A7BIwJ!(?xBh=`h=}B?xBh>`h>}BCIvS!-^h9w&zza#7Bx6h9bpzx^h:w&zya#7Bx6h:bpzw^jou;!xl@c>#xk]jo4@GzoKu(w>%(+-@`_y4Bbm|Hasbh}.Aq]cjsB]nvx5h1>B]n|flw=]nzIu!!#]o&]h5Ow2]nxM[]lYJu^C#2a%B^h-v8!&avxJh1}y7avxJh5}~<]o#IvS!#]o.]o,5xSh&Tv8A&+12}K??@>bu}EC@;E7bpzhbpzfJu&A#)}H??@>bu}Gavbp}D9]k;xYh.oJp^u)uyv=w-----.a)a.a8a;a>aEaHokoeof]o!Iw,!%B]nwCar]o1x5h2>B]n}AIwR!%;]kMar]o0M[]lZflw=]n{Iu!!#]o']h6Ow2]nyJu^C#2a%B^h.v8!&avxJh2}u7avxJh6~#<]o%IvS!#]o-]o/JrD=#'1@@]fX@Ivu!#bj|Rx>ezbtz?fmY]k}OuJp4d9s+tvu?uav^w/D.a'a)a5a7a;a>9azb6b8bhbwIwU!&faw8]iTsd@bjzOJu&E#&(Pv>rif_WrgtW>Hzegtu35]iS@`_yeKu^vI%a+a@aEIwC!+7Iu9zba#>@Fza4^c~w_!#Ebuzd:<]gHAboz`Iv~!~.bozc@=pRc]%'|T6Fxa7Ax&bZtu!'x2hI@]m^]m]JvVA#%a+ba|vxQhMs<8#'*>B]i_:@]o@zK_z6@au]o3ba}B]k(QwQA]mfOw2]mear]m5fcu0]mgi-uhv1]iyJuOB#+-Iww!%`_y5t[5EtpPuorkIvH!%axH{2gqri6Ivw!+78B4@@>btxjj7Urx[we]f#d@pis+tvuIuyvKvzwDwh3a&a)a1a:aAaOaXa[ak>awa|bMbYb]bjbq@Gx`4@]lDIvp!%flu5pNGzU@Iu(!#Etraw7rqIwt!%@5pOtXJs+as#&(@Gz^OTru4Gz_Ivv!&fjw;]l^`_yfRwX7pMc;#,@Ivw!#]f>]f=;au]jiIw7!1IZ!)@=x6iU@]iU>?]iO@:]k-IV!%5@roo[IvX!%>=rw8`_z7ctttuywV'+3a4a6a?@@>btzS>D=4@@>btz[ibv_vM=Ivw!&78H}p:9asH}oCbqzZ:^d)uy!!sj>=sej%YXw3]fGJu[B#2a%IZ!)@=x6iV@]iV>?]iN=9rs@:]k.5@`_y6Ju]@#')>B]i8Ou)rm:x7jw]jvIv%!%@G}v;pPi*uKu/]ltd;pUrGtvu?vbEbIbKbLbX@Gx_4@xEo8]o94Abgx^Ivy!&fawC]locxumv]w/B}8'+1a+a.aM4??ba|L>Bas:=bf|#Jua@}7|{a#>?H|}x9evbn}&IwO!&9bozAB@bh}sIwL!&@`_y7IvS!*=Iu!}ia#`h5}B=Iu!}ja#`h6}Bi1w{uD]lncxu(vGvzA(*a/a2a+a5a:fcs@SIu]!/IuE!&4@]n&7x@g4]geNv_]eI@`_yh?8`_z9x6gL4BbgzW5@`_y8d:tvuPWv=ZwVwizp.1a%a.a.a7a9aaBglwE7r}Iwp!%Pv>s%tL>Bs!Iw8!&B@bezV4s]@`_yk5EtKi/rv[]f[?8`_z<5@`_y;Iv;!#]dkat]dj",
- 11_775,
- 1912,
+ "!#&})%##u%}'&}*%%~!C%/~!J~!J~%L~y<.)~!J~~%<#~*_~*<~~#GD~#p~~#y:%#-'}m%##%+#.%%}'%})%#%##'}%&}(%}%&'}#&%&#&#%%#'#'&&%##%}8%%}#)&%('&%((&&%%&&%&#%%%%'%(#)-'#&'##%(}#&%%%3-)&,#%##&)')&6)*''%%%#'%#'%(&)4(&#(`%11&*#0#;&56/%&%%%(.B1'%T,%,+6A##%&%%#&#%%#/2%'%:K]),3~!+2(.jF:~!.#10>N>%%%~!J~!I~!E#Wc1%#~!G~!J~!H##~!H&E~!%%~!%~!N:+~#pF~!'~##~!p~#U~!1AAX[~%.~!R~'2J~&5'~!D'~#l~!n~#I%~!G#-~#3~!|~!@*~!JOx~!xOJ5~)W}#%}*&}#%##%##%##&#-}&'#'&%#.++}%m:2,#,@&(}*%}*'%&##&#%##%}&0}#.},U},%}+%}&%}#%##&}B%(}(%%}%%##%)})%##%#&}&%##%&}<%}>%#%&}3%}9%}/%})%}*%}*%}?&}&%}3%}&*#%})%#%#)}#&#-#+*%E%%'%#%##%}#*W#&##I}#&&##%&%##%#%Qc&%%))w/0%%&#&%#(#.%-''''++++7}>%%2'',##1,#%#&%##&#'##&#*#9)%&%}#*}%,#)%U%A&%#'&}#)pSD',9E&-0#MN#@}(+}&%&~!:(~!X}#*}(&&}(&}(,%}%&#+&}#&}I%#%}%)#(},'%#*}4%%#%}(''}#/##(##),%*&##%%)#&}(.}&%#&}%%}*&#%},&&}&%}#%*'#%})%}D&}&%}-&}6&#&}-,%}#%})-~',~%L~!5~#A~%/e~!5h~#U~#p~#q~#q~*P~8?'9%~!,#%})%})%}@%}?%}(~!?~#<~#pP'~#>##E~#=1#%K+~#?#~#pJ)~#A~#mF1~#A'~'X%&#~#lR~#N~'N~#r~#m#-~#i'&)6%(%~#B%##%)&%#~#_%#0%~#]5##32~,w~2+#:&#%&'0%&>%}#>##F+)#%&&#(+_}4&}-%}(&}@&}O7Fdf0@+/v4}&WS%##&/0#&'('B#%}.%}'+#%}#%%&#&%#%##+#&#)#6#'#.},%}c%},%#%##%&#&%#&~#>'*-.}%%##%}#%%}%'~#)D/%}#%*&~#_}%'(~#S2##'.}#~#=##*'4%}&')}#%&~'E%%,#&~#B.5}(&'%#~#O1##%&#&#+'~#8*;%##%%&#%(&~#;#@%}#&%##%(#(~#H1}'%&}#&&~#?A}&'~#D#%31}(&%}%%#~#[}'%&#%}'%~#9C##%}%&}(%~#=&%,3}%'&%}'~#]'#&&)}#'~#Y%-(#~#^-%'~#^%%&#&&&}#~#b~2t*&'~&(~&@~0%~e~3}%*''0})&}+~!8}#-##uD*)1~!,5/~!+#40~!2*4~#X:~!/*7~!.#~#p':~%x&~!H~&7~!I#~!+A~#p'~!F~~#2':5~%e>~%Mp)~#r&~#?:5~!~-~!9~!1U&#~#*~#rTlVge;~!3~!&%#&&#&}#%},%%}'%}+*Q#%}#&}(%}'%}<%}#%}%%'}'%}:;1~#!~!2vQ(G~!=%~!=*A%-%##4ZM-#)&s1&&0f'&##%'#,+(B31&?,,(#'#Lu96Y&,H5W'7)+o54#JI*#<)o)0.D07,>-.G%(#QDF4*-p&].#0[%H11k0)-#r)~!16^#6Ol/%<+0vE~##A)~!/=%66G1UM.y1%+D:8PG~!9#^3~#K.~!Q~!.`2LMUc0eOF~!-~!i~!.vH*~!|3+q7h~%+s~!7~!r`%#2}%(=MFE~!-1.Q~!74m*U.-~!:,Y-%'#)-7/9N'*<)F516;%&7#%)%5(%#&#)%-*)*%#%'+)&&/#)+#'&&-%#'%+%%&#%}%'}#*#wCx-jov!hHwjw[w@hVsK~t9g:vJgHufig~s~uRhNuN~z4~qq3~|~p!!UU!a#?RARTyC!%?Z!a!VVU!e9#b&bXc^d.dDdFdKdMdRdTdUd_d`dkdndodpdsdudvdwdxdydzd{VUd|d}e!e(e*e/e0e2e5e7e8e:emfejlkll%l)m|o~p!p%p&p'p(p)e+#VU[__`'+.Ua+a/Va3a6a9asrKw7r83RnVGv~W<;r<5ZzOkcx9wlspv7veReig?uzpUGwH[]yQg?vaRh=~{pS:9pTdl#!a)!a+a>_aBUaGaJ!aMGw7!'jBvNwF7Rg_cH#a#Roq43Rj%DtVHw7A!&,hFwV4Rg{j[wwvKwMRf11ss]y}=5ZzPg;uhs_2E~ihRuc>Rh7e)#VU%__!(!a7aUaXaf!ahajU=Ub6UbZb^!bc2Dtmafaop;HvzDW0~wr>xXiVl=Ov)upw9v`v7OaYRfaCaG7cFx]f@Uaya]r~a^pa~r4pYHv-DW'MverP?2pZtscqumvo@^'a4aHDtj]z#hQu`3Gqq!)kAOq`wyQ4Rkil&x;vOOq`wyQ4RkKHwAB[)5ZzSfxvKRgMj_u^whusRf52E~ne%#_[_V&+.a0!a3a7V`a9Ua>ar12DtIp2gAsCxSd0tANw~4rTHv0DW'~vrX?2rRtXaGh;shDtpGw[W4axs^r(g?sqrYXevk@uWwhvwsl4ReuGwAW5Rewk_vvwdOv+ubRjuGw[^E~eMv?r[:=c6#*[1Xmq~yi9=Hqdal!(.jrvguU~sRlljdqtsnvtXl{~xxIfv1Xls~xxIfs1Xlm867A1??!a'a.a>g;w^GqV!&NPCRl4~}Rl6~}Rl5g;w^GQ}f!867A1??=RlADtbGvE!(jfvXvwsl4RegjCvbwD5Rf6]z)j[wVvBwVRg]=5ZzY2E~hs|dHr~uXvnwiBW(a)au3cicjcl2DtP~r4rwHuVDW'Mver{~vrytcHwLC!a;aWidv7uhHqgal!)a*k9v/vXvwsl4ReC78Gv^!#axai=12bh~j4?Daj78bq~kgGuKGqO!.?41A4?a]?41A4E}u4@@ab4@bu~?h5vb4qx]z*IvZwf!&a%0Ns;axRehkzudv;vbq_sFuXr.5Re~y/ou#0a0!aJ8`a|!bN!bX!bmb~c@!HGww!'j[w;ugwdRhGhUrw=RhRl!&1?TmqJ~yin@xwhSqDq[aj!&,0a-~yhUjGs;ufXh]g;wTThOJjpv]qAwyOTntJ~{RhXigwiuKGqO!+k{s;ufqFudwYw9To;JjVwTuTwTTo:Jj_snuKwJSheqY!'~uToGJjpv]qAwyORicGv0!*kyw9uevAvSv`YgWk^w`w5s?uzu`Simq>!&1?TmrJ~yioGwy!a2jJQq^BGvy!+gFw]{,hsJ~yieigwSw]{,htJ~yifHsm=!+asluov3w@w{['a-a:a]a`abbub{b}c#KwCRlFa]p?Hv^?W'~ws*6Rl1?xbf[9RlJHuVDW'Mves.~vs,tfxdf)hJwS4Gqm!0Gw1!&iguawdRgVjtvJs[v5vXRg&kxw*v8v4w6wt:Rm.Xf)b1}WGw?!)k;u`w3slu_Yl/Mx-SfNqM[1Xg;jdw^w2PCRf{j@vJvb6Rj'[1Xmr~yi:=Hqdal!(.jrvguU~sRljjdqtsnvtXlw~xxIfu1Xlo~xxIfw1XlnGwxW5Rf*jqqIw%v4wJRm/~qsJ~pRg6Gu~WXf(Rfkjpq;vFx4uKRmze)#VU+__!.!a'Ua9!a=ab`ae!al!aq!b2b4b8GsgWg!x5toDtnh/skDtr~r4s0xsoRuVx8W'*Mves6~vs4?2s2tg]z.gCwhIqNqn!(.a*jgvgw2PCRfOjdw^w2PCRfL~}RfNjFw2PCRfMg=sCt&jkvKv(squ`Rga=5Zz[GwzWYgbKudxpkJqZak!)a5kaufwSsnv7veRhwBGvy!*gFw]xLhs~yhuigwSw]xLht~yhvhSvo;Rhx2]y[1XiJIsmw%!,aDaFxaiT4AxLiT~yhkGu~!a0hJuK@xph`qYaj!&-~yoHjpv]qAwyORhb~{Rhdaj71F~)RgZSiUwJ!+NueAxLhh~yhl4YiUe'#_[)_V!a&!a,Ua9!a=a_V!2U7;g*qSpnfxq?Rf,GsjW2DtQDtlGwp!#qwt'HuVDW'Mves:~vs8th]z/c{#`-cb#!'igvnudRgz1sxGv^!(jBvwsl4TegeBiWsFuXReALuYxph%qCaj!&-~yh,jhvKw*sK9Rh.~{Rh1=5Zz]jYvLq;weReoGw[[]y]Mv?sV!aBaEar6!bQbSbUbVGw7W~wpj?x]fZg9w=Rlf?GuX^DtSay4sBGx8W?2pktihGsB2sF]z0Nw}4piKw7s@Gv/!a54?GqT!a'GQ^?r)12Gv=^RjT4YjOk5udwdu]v6@RjR<;xNiGLwVRhrGv~W<;sH5Zz^dCq8qeuHwQ!a(a/a7}La@aIa^NPCSfMq9[1XlHjgvgw2PCRf|jgvgw2PCRfQkCv8v4w6wt:Rm-44xHi,NPCRfa<;t(g?uzsD2]y^~{s>:9pldHsYuJuowPay[)+0aZa_adaiKv%Ri11XotDtWKv%xYi/RopGw9^RiEHwZD[a71XeLxXeLg9OIqMqj[(.1Xgig?ubr,jdsFsHvtXl#~{Rh)jjv3qasFuXReB]z1=5Zz_2]y_hIwF7Ri0cquXvn@W(,0g?sqsJg:u[RiD]z2=5Zz`2]y`d[#U&V!'`,]z3s~=5Zza2]yadHqHs+umwPBW'*a%a*a.a3a82Dtu2DtM2Dtt~r4pmGx8W?2sLtq]z4=5Zzb2]yb:9sNdCr|uJuowPW(/a'a4a7a;2Dt[~r4sOGx4WMvesSt]RiQcH#&44RiA43xVj#4Rj#?axxbjOg8v@RjPGx:W;bj~[twgEvuReRde#_!'U,!.!a'KwVxfg{}.hUx=ayRmZ@bl~T;B1?bh}Ej_s?uzu`xpkQvE?[)gCvgRkVg;w^RkXhNv!YkSaxRjbGp+!)Gp,^RkHRkGaIRkI2axRkFGvm!){0aReqgHx#ThFeqYj.dJ#`&U,/5Zzdxbi,<:Ri,gGu^RiLdSqGqsuIvTwfx)!/a4aCaZa_an7:>ayb)IqXw>^%'Rk+Rk(Rk*Rk'xsk#qlwr^%'Rk:Rk=Rk8Rk;IqXw>^%'Rk1Rk.Rk0Rk-xzk%qPu{w>^%')+Rk@Rk7Rk4Rk?Rk6Rk3^%'Rk)Rk&RjxRjwxsjuqlwr^%'Rk9Rk^%'Rk/Rk,RjzRjyxzjvqPu{w>^%')+Rk>Rk5Rk2Rk!Rj|Rj{Gw~Waybh}#g8Qp8IuXvo[)-]ye:8Ree:xfh&~'9Saru}^RmmgFs]Rl*cC#(9xTeUYeU=Sh7uS^RoFxih8}Bdr#!aWau2b#!b'b5UbnUbuUc3c7c:cRh@4GuH!0NPCGw>[g;w^RfqhNv!Yfrcqq]sl3!#a#(,r0Rji@Yi!g?sqRh~Kv%Ri#hSvbYn(83Roxg9w=RmjgHwGxckq8Ykqc~#)a8!aE<;xTaOxih=|tGw%[1xbaAr%SgNvG^bq}`4Gx1[g;wdRgN4bu|~c]#!(xSh.^Rl`Rl]GwQWXia2RibKwCx^foRlcy,gp#'0_Ua)`a,Ns9=RnOGwoW=RnM=RnQGwQ[?4bf~SBbf~R44RiRhJuw4RiS4;p6j@QPCGw>[g;w^RfohNv!Yfp4bg~dGv.!'<;8;F|u;YgwhQw[DRj9e9#VU%___)a:aEaMa_ap!4RnndA#Va!&pAA1t/hUx=ayRm[Gw=[gFw`Rm^RfzRfycquVwLay!|y0a)a-:xyiH#`(;3xaiHgHwaRkshFvW1tB8;Ritx{bx#V!,34xBbxjDv7ua@RiK;by~n2Du;2Gw@[?;Rj5<=Rj+cqw#w|CW(a9aELQqz5ZzfxsczvSwQ!(-0>xSh9w(vs;Gw>[45F~B867F~Ac>#(ax1?9Rm{Mv?rIGw8[aIVaVakAa}b+b>bcbobyb|Uc!Gvi!~9u7GwL!#s!=Re;Gv~W<;rO5ZzgHO@!'+?xaiY9RmvB@Rni8Sd4x%W<;t0tEIwHx)!)a1aHGvo!~G?2Rh?GvJ^bp|sKwdGvH[AXo1g;wTRo0dI#U[`)9@r#@YhDayxKhFaYRnoirQvNRmxGrx[whvbYn'Gvj!aWGwH!aPd1#a(a0a;a>!aFd2#a!%&(!*pMRf@pLRfBRfFRfHGp-^RfARfCcx#a!U%pNRfDRfIaJRfEGp.^RfGRfJaMRfK9RedC;Rj72]yhe.#___`&a*a.a6a9afaj!amaq!as!axb(Ub-b0b;!bkxYhLRo'HvR=W)~wsW:1xed3}wRn}Nw~4rUGx8W?2rStyGw1|d!94@bu~88bp|fGv`!(jGvfw->ThN~!bS~ndLrusmuovpx]z<@Gx,!&hFPCRlUhFPCRlVcqvPw'?[)a0a4?XgLgGw`Rh#axGw>!(jdw^QPCRfd~qsJ~pRfe5Zzjg8QReKHvDA[)]yi1@bk~OMv?r]Gvy[gHvKRecg>v`ReHdl#!&!.a,a3!a7aQUaTawbEbAbM!bObabf~r4p}Sekx8W?2p~t~Gx0^Dt{29p4Gw:|>!Zz=Nw}4p|xpf?vbKv&sX=5ZzlGuX[]ykMx5uAg@x5u=df#!'U.a'a+!a.Va1`a6gDsFxdd6tCGx6W~vrku#]z?NuZ;rl2Du/2DuE=5Zzm2]yle3#0Ua)Ua-___a1b,bGbTbkcJcPcR!cU!cVcXc[cfcr!cscxc|d'd(HQAW&?E|hE{tKvJRlMKwCRlDxVhKRo&1Xl}de#[!*V!a%0!a/a1a4!ab~wrniowfx*Rm_?1bq|Xg8s:t46Sl.vE^RmEbh|ZRn|BW/a&Mverrd+#V%89rpbo|Ube~_u%Iw+wSW)a*1Rl^BQRm&hWs>Xlh7Rfly*hI#|[_!ag!aqAcqu|w>A!)a,a3aTNPCxbfL1bl~Fj>w(vs;Gwr[gCvgRft=Rfsjmw^QPwWRf~hNv!AHu|@!'.NPCxkfP|P1?=<<;bu}*>B861??Gw/WAE|R6AE~/AE|S8bp|TdN#U&V!*gFw`Rm9<[45F~(hNv!YfgHul9W'Xm=ZznB@Rn8hNua@Rn>c=#&@Yg`bd}3xmkZ#}HgBu[RkZ1?xYa=YmGdi#[V+V0VUa'?E|?Xnp?4bh~+gAwJRiMKwCRm5hYwJYnrGv,[1XmJxmkY|=}G?Gwr!&hWs>Xlgg>QRm%Gv`!(jGvfw->ThM~!bS~ne-#___V&!aRa[a`ah!alaoUb(!b/b3|FUb:!bEf{weRh!da#_a#`/!a9?p@Gw]^RkoxTk~@4Rk~xafbA8Xoy!1?bd~N:xTh*xih-|11E{r@BGvy^bh|%bh|&Hsm=!a(a-xphiuS@!|(a#To[J4A{.hgen>xihm~m2xThfbt|'xphjuS@!|)a#To]J4A{.hhen>xihn~mdY#WU'V!)bo{qLuYq%bj{yjAv]vH4Gw>!'g;w^xTilbt{vhNv!AxTimbt{}xZd8Sa8wJ[?@[)-0a/a4aIaVKv%Ri3KwCRl@=Th6enKv%Ri2Gw]WThJenTaSenhSv1;RmuHuRA[(?Xl>ThIen{1aQen84Ti9enGwX[?Xl?Mu^Ti:en8:Th%enHru;[0?Xg1?GwReY]zDdB#U[!)xdd=t?:1F{I;4Rkmxrd9#!(j?uqw@axRiXt@Gwo!-;Gv>RoK8:Rij8bp{l:4xkeZ{hHry@~g!~ibs~cHun={j!a'HO@[)LQRj:g?ubRj/gHusRj0xbgcbr{iMvFRi6d@#V!&]ypt;hSwH=Re@dc#U}RV!&+U/!a)]zE=5ZzsNvU4Ref2]yqHuVW/a&Mves/d+#V%89s-bo{cbe~_u*IvDwRW)a%1Rl_hIs>Xm(Bwi!)a,a@aHaRNPCxbfN1bl~:j>w(vs;Gwr[B861??ZztB@Rn9hNua@Rn?Gvx!'?xVa>YmHiqv4wdRn*1?E~MIslw0[{[)gEvuRe`]yrGwp~1!Bbr}Vy!h`ryuXvbwQD^,.a(a+a:a?aBRoJca#WRoN<;s7Bbh{bxSoH89s5?2s3Hry@^&RoL=RoP8:Rikiqv4wdRn+8bp{`u+dA#V!*:1Sd;x!Ut7y+h%#&!)V-!0!a(!a,xihk~[4>xiho~`:Ro^Gvy^RohRof2xwh`slv`@!'a%/~*==?bt~#6Rkty(hhp*qaFaJpIpCpDRo]GwPWYoTBavRokxShlxihl~W4>xihp~b:Ro_Gvy^RogRoiHru;[/?Xg4?Gw<^bn}0x]fUbxyLgIQRlZLuypodq#,.a0a3a8!a<8V!b!b;!b@br`b|cp#U&g=w]Rj2t8?bn{!HuVDW'Mves;~vs9u,8:Rh%@bs{:GwF!~mbs{=?;q1c`#%}13bhy:4@xDbZ#!'xPi'Xn!*45A1??^RfvRfu9axRkDGw[!/Gw@!(?;xTj3Xj3<=Rj-?8RkeGOW2?sAp:Gv~W<;sI5Zzwdl#U'`/`a5VUa8!aD??w(vs;Gw>[45F~Q867F~PBbu{+8xxd<#U^t><;t9j^QPwWRg!Hv.A!/a'Gw@!(?;xTj4Xj4<=Rj,;6sE?8Rkf2]yuHv/?[({0ho~!To`~!jHw]ub>{0hp~!Toa~!Gw<[4Abd}{jAv]vH4Gw>[g;w^Ri7hNv!Yi8DtxKv%Ri)HvF?!-0SgnuW[1Xi@>RhBLw!RipGwZ!{*1E{)]zJA?bl~Z@BGvyWThgenThhen=5Zzx?Gw+!{?]yxGwf[9Bbu{E?bl{Adl#!-!a)a+Ua/V!a2`a7!a<2d%#U[A4q0u9Gx8W?2sMu5;p7]zM2Du@=5Zz{2]yyGvR^Du89q2dq#!&a%0a/!a3a6V!a;`a@`aE~r4sPGx4WMvesTt} no value)
- * 13 FLAG13. If valueLength>0: semicolon required flag (implicit ';').
- * If valueLength==0: compact run flag.
- * 12..7 BRANCH_LENGTH Branch length (0 => single branch in 6..0 if jumpOffset==char) OR run length (when compact run)
- * 6..0 JUMP_TABLE Jump offset (jump table) OR single-branch char code OR first run char
+ * The trie is a flat `Uint16Array`. Every node starts with one header word:
+ *
+ * 15..14 VALUE_LENGTH Number of words the value occupies, +1.
+ * 0 = no value; 1 = value inline in bits 13..0;
+ * 2/3 = value in the 1/2 words after the header.
+ * 13 FLAG13 If VALUE_LENGTH > 0: semicolon required ("strict"
+ * entity; `;` is never stored as a branch).
+ * If VALUE_LENGTH == 0: this node is a compact run.
+ * 12..7 BRANCH_LENGTH Number of branches (or run length for runs).
+ * 6..0 JUMP_TABLE Jump-table offset / single-branch char / first
+ * run char (see below).
+ *
+ * Branch data follows the header and any value words. Its shape is selected
+ * by (JUMP_TABLE, BRANCH_LENGTH) in the header:
+ *
+ * Single branch JUMP_TABLE = the only child's char, BRANCH_LENGTH = 0.
+ * No branch words; the child node follows immediately.
+ * Jump table JUMP_TABLE = first covered char (> 0), BRANCH_LENGTH =
+ * table length. One word per covered char: 0 = no branch,
+ * otherwise the child's offset from the END of the table,
+ * +1 (so 0 stays the no-branch sentinel).
+ * Dictionary JUMP_TABLE = 0, BRANCH_LENGTH = number of branches.
+ * ceil(n/2) words of sorted keys packed two per word
+ * (low byte first), then n pointer words storing the
+ * child's offset from the END of the branch data.
+ * Compact run VALUE_LENGTH = 0, FLAG13 set. BRANCH_LENGTH = run
+ * length (3..63), JUMP_TABLE = first char; remaining run
+ * chars packed two per word after the header. The target
+ * node follows the packed words immediately.
+ *
+ * Pointers are end-relative (rather than relative to the pointer's own
+ * position) because that makes the common "child encoded right after the
+ * branch data" case a small constant, which compresses far better. Offsets
+ * to already-encoded (shared) nodes wrap via uint16 modulo arithmetic; the
+ * decoder masks navigation results with `& 0xff_ff` to match.
*/
export const enum BinTrieFlags {
VALUE_LENGTH = 0b1100_0000_0000_0000,
diff --git a/src/internal/decode-shared.ts b/src/internal/decode-shared.ts
index f4ed62cf..ba0aeef5 100644
--- a/src/internal/decode-shared.ts
+++ b/src/internal/decode-shared.ts
@@ -1,107 +1,222 @@
-/** Number of most-frequent values assigned to 1-char codes. */
-const DICT_SIZE = 61;
+/*
+ * Inverse of the encoder's SAFE alphabet (0x21..0x7E minus 0x22, 0x24, 0x5C),
+ * precomputed once at module load. A flat table lookup per input char is
+ * measurably cheaper during import than recomputing the three exclusion
+ * comparisons inline; entries for excluded chars stay 0 but are never read.
+ */
+const BASE91_INVERSE = /* #__PURE__ */ (() => {
+ const table = new Uint8Array(127);
+ let code = 0;
+ for (let char = 0x21; char <= 0x7e; char++) {
+ if (char !== 0x22 && char !== 0x24 && char !== 0x5c) {
+ table[char] = code++;
+ }
+ }
+ return table;
+})();
/**
- * Decode a dictionary-encoded trie string into a Uint16Array.
+ * Decode a dictionary-encoded trie string back into its Uint16Array.
+ *
+ * Stream layout (consumed in this order):
+ * 1. dict1 atoms — `dict1AtomCount` uint16 values, delta+RLE encoded.
+ * 2. dict2 atoms — `atomCount - dict1AtomCount` values, delta+RLE.
+ * 3. dict2 ngrams — `ngramCount - (dictSize - dict1AtomCount)` entries,
+ * each a pair of slot codes that resolve to earlier slots.
+ * 4. dict1 ngrams — `dictSize - dict1AtomCount` entries, same shape.
+ * 5. data — slot codes, each expanding to one or more uint16 values.
*
- * Format: [dict1: D values delta+RLE][dict2: remaining delta+RLE][data]
+ * Codes use a 91-char base (printable ASCII minus `"`, `$`, `\`):
+ * - char1 < dictSize → 1-char code, slot = char1
+ * - char1 ≥ dictSize → 2-char code, slot = dictSize + (char1 - dictSize)*91 + char2
*
- * - dict1: D most-frequent values, delta-encoded from 0 → 1-char codes.
- * - dict2: remaining unique values, delta-encoded from 0 → 2-char codes.
- * - data: each trie entry as 1 char (dict1) or 2 chars (dict2).
+ * Slot index → token kind:
+ * [0, A) dict1 atoms (1-char codes)
+ * [A, dictSize) dict1 ngrams (1-char codes)
+ * [dictSize, dictSize+D) dict2 atoms (2-char codes)
+ * [dictSize+D, end) dict2 ngrams (2-char codes)
+ *
+ * Both atom dicts decode before any ngram, and dict2 ngrams decode before
+ * dict1 ngrams. So every ngram entry references slots whose contents are
+ * already filled — no forward references to handle.
+ *
+ * This runs on library import, so it is written for a cold VM: flat typed
+ * arrays instead of per-slot number[]s, indexed loops instead of iterators
+ * or spreads, and slot contents stored as either a plain value (`single`,
+ * covering every atom) or a range in a shared `pool` (ngrams).
* @param input Packed trie string.
* @param resultLength Expected number of uint16 values in the output.
- * @param headerLength Number of chars occupied by the dict1+dict2 header.
+ * @param atomCount Total number of distinct uint16 values in the trie.
+ * @param dict1AtomCount Atoms in the 1-char range (`A` above).
+ * @param ngramCount Total number of ngram entries (dict1 + dict2).
+ * @param dictSize Number of 1-char code slots; the rest of `BASE - dictSize`
+ * first-byte values are 2-char codes.
*/
export function decodeTrieDict(
input: string,
resultLength: number,
- headerLength: number,
+ atomCount: number,
+ dict1AtomCount: number,
+ ngramCount: number,
+ dictSize: number,
): Uint16Array {
const base = 91;
-
- // Build base-91 lookup table inline (91 printable ASCII chars, excluding `"`, `$`, `\`).
- const lookup = new Uint8Array(0x7f);
- for (let codePoint = 0x21, index = 0; codePoint <= 0x7e; codePoint++) {
- if (codePoint !== 0x22 && codePoint !== 0x24 && codePoint !== 0x5c) {
- lookup[codePoint] = index++;
- }
- }
+ const inputLength = input.length;
+ // For 2-char codes, slot = char1 * base - twoCharBias + char2.
+ const twoCharBias = dictSize * (base - 1);
let pos = 0;
+ /** Read one slot code at `pos` and return its slot index, advancing pos. */
+ const readSlotCode = (): number => {
+ const c1 = BASE91_INVERSE[input.charCodeAt(pos++)];
+ return c1 < dictSize
+ ? c1
+ : c1 * base - twoCharBias + BASE91_INVERSE[input.charCodeAt(pos++)];
+ };
+
+ const dict2AtomCount = atomCount - dict1AtomCount;
+ const slotCount = atomCount + ngramCount;
+
/*
- * Delta-decode helper: reads `count` values (0 = read until `endPos`).
+ * Per-slot contents: atoms (always a single value) live directly in
+ * `single`; ngram slots hold -1 there and expand to
+ * `pool[start[slot] .. start[slot] + length[slot])`.
+ */
+ const single = new Int32Array(slotCount);
+ single.fill(-1, dict1AtomCount, dictSize);
+ single.fill(-1, dictSize + dict2AtomCount, slotCount);
+ const start = new Int32Array(slotCount);
+ const length = new Int32Array(slotCount);
+
+ /**
+ * Decode `count` ascending uint16 values from a delta+RLE stream into
+ * `single[off..off+count)`.
*
- * Encoding per delta:
- * code < 89 → delta = code
- * code == 89 → RLE: next char = N-2, emit N consecutive +1 values
- * code == 90 → escape: next chars encode delta-89 (2 or 3 chars)
+ * code < 89 → delta = code
+ * code == 89 → run-length: next char encodes runLength-2; emit `runLength` consecutive +1 values
+ * code == 90, next < 90 → escape: delta = 89 + next * BASE + after-next
+ * code == 90, next == 90 → double-escape: extra char for very large deltas
+ * @param count
+ * @param off
*/
- function decodeDelta(count: number, endPos: number): number[] {
- const result: number[] = [];
+ function decodeDelta(count: number, off: number): void {
let previous = 0;
- while (count > 0 ? result.length < count : pos < endPos) {
- const code = lookup[input.charCodeAt(pos)];
+ let slot = off;
+ const end = off + count;
+ while (slot < end) {
+ const code = BASE91_INVERSE[input.charCodeAt(pos++)];
if (code < 89) {
previous += code;
- pos += 1;
- result.push(previous);
+ single[slot++] = previous;
} else if (code === 89) {
- // RLE: next char encodes count-2, emit count consecutive +1 values
- pos += 1;
- const runLength = lookup[input.charCodeAt(pos)] + 2;
- pos += 1;
- for (let r = 0; r < runLength; r++) {
- result.push(++previous);
- }
+ let runLength = BASE91_INVERSE[input.charCodeAt(pos++)] + 2;
+ while (runLength--) single[slot++] = ++previous;
} else {
- // Escape: next char(s) encode a larger delta
- pos += 1;
- const next = lookup[input.charCodeAt(pos)];
- if (next < 90) {
- previous +=
- 89 + next * base + lookup[input.charCodeAt(pos + 1)];
- pos += 2;
- } else {
- // Double escape
- pos += 1;
- previous +=
- 89 +
- lookup[input.charCodeAt(pos)] * 8281 +
- lookup[input.charCodeAt(pos + 1)] * base +
- lookup[input.charCodeAt(pos + 2)];
- pos += 3;
- }
- result.push(previous);
+ const next = BASE91_INVERSE[input.charCodeAt(pos++)];
+ previous +=
+ 89 +
+ // eslint-disable-next-line unicorn/prefer-minimal-ternary -- branches read a different number of side-effecting input bytes
+ (next < 90
+ ? next * base + BASE91_INVERSE[input.charCodeAt(pos++)]
+ : BASE91_INVERSE[input.charCodeAt(pos++)] * 8281 +
+ BASE91_INVERSE[input.charCodeAt(pos++)] * base +
+ BASE91_INVERSE[input.charCodeAt(pos++)]);
+ single[slot++] = previous;
}
}
- return result;
}
- // Decode dict1: DICT_SIZE values, delta-encoded from 0
- const dict1 = new Uint16Array(decodeDelta(DICT_SIZE, 0));
+ // Streams 1 & 2: atoms decoded into their slot ranges.
+ decodeDelta(dict1AtomCount, 0);
+ decodeDelta(dict2AtomCount, dictSize);
+
+ /*
+ * Streams 3 & 4 are read in two passes: first collect every ngram's two
+ * references and derive its expanded length (each ref resolves to an earlier
+ * slot, so lengths are already known), which sizes the shared pool.
+ * `order` records the slot each entry fills, since stream 3 (dict2)
+ * decodes before stream 4 (dict1) but occupies higher slots.
+ */
+ const references = new Int32Array(ngramCount * 2);
+ const order = new Int32Array(ngramCount);
+ let poolSize = 0;
+ let ngramIndex = 0;
- // Decode dict2: remaining values until header ends, delta-encoded from 0
- const dict2 = decodeDelta(0, headerLength);
+ /**
+ * Read `count` ngram entries (each = 2 slot-code references) for the slots
+ * starting at `startSlot`, recording references and assigning pool ranges.
+ * @param count
+ * @param startSlot
+ */
+ function readNgramReferences(count: number, startSlot: number): void {
+ for (let index = 0; index < count; index++) {
+ const slot = startSlot + index;
+ const a = readSlotCode();
+ const b = readSlotCode();
+ references[ngramIndex * 2] = a;
+ references[ngramIndex * 2 + 1] = b;
+ order[ngramIndex++] = slot;
+ start[slot] = poolSize;
+ const entryLength =
+ (single[a] < 0 ? length[a] : 1) +
+ (single[b] < 0 ? length[b] : 1);
+ length[slot] = entryLength;
+ poolSize += entryLength;
+ }
+ }
+ readNgramReferences(
+ ngramCount - dictSize + dict1AtomCount,
+ dictSize + dict2AtomCount,
+ );
+ readNgramReferences(dictSize - dict1AtomCount, dict1AtomCount);
+
+ // Second pass: concatenate each ngram's two halves into the pool.
+ const pool = new Uint16Array(poolSize);
+ for (let index = 0; index < ngramIndex; index++) {
+ let write = start[order[index]];
+ for (let half = 0; half < 2; half++) {
+ const source = references[index * 2 + half];
+ const value = single[source];
+ if (value < 0) {
+ let read = start[source];
+ const readEnd = read + length[source];
+ while (read < readEnd) pool[write++] = pool[read++];
+ } else {
+ pool[write++] = value;
+ }
+ }
+ }
- // Decode data
+ // Stream 5: data. Each code expands to its slot's stored values.
const out = new Uint16Array(resultLength);
let outIndex = 0;
- while (pos < input.length) {
- const code = lookup[input.charCodeAt(pos)];
- if (code < DICT_SIZE) {
- out[outIndex++] = dict1[code];
- pos += 1;
+ while (pos < inputLength) {
+ let slot = BASE91_INVERSE[input.charCodeAt(pos++)];
+ if (slot >= dictSize) {
+ slot =
+ slot * base -
+ twoCharBias +
+ BASE91_INVERSE[input.charCodeAt(pos++)];
+ }
+ const value = single[slot];
+ if (value < 0) {
+ let read = start[slot];
+ const readEnd = read + length[slot];
+ while (read < readEnd) out[outIndex++] = pool[read++];
} else {
- out[outIndex++] =
- dict2[
- (code - DICT_SIZE) * base +
- lookup[input.charCodeAt(pos + 1)]
- ];
- pos += 2;
+ out[outIndex++] = value;
}
}
-
return out;
}
+
+/*
+ * Warm-up: decode a minimal trie (one atom with value 5, one data token) at
+ * module load. The real call decodes ~18k chars in a cold VM; this dummy
+ * call makes V8 baseline-compile `decodeTrieDict` first, which speeds the
+ * real decode up by ~15% (measured on fresh-process import). Intentionally
+ * not marked #__PURE__ — dropping it would silently undo the effect.
+ */
+// eslint-disable-next-line unicorn/no-top-level-side-effects -- deliberate warm-up, see above
+decodeTrieDict("(!", 1, 1, 1, 0, 1);