diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 38ed5ab..0053df1 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -158,14 +158,17 @@ jobs:
#
# What gates here (#260): the FETCH — a hash mismatch is a hard failure, or
# the pin is decorative — the VALUE census against the committed `corpus`
- # block of test/fixtures/explain/values.json, and the OPERATOR IL-head census
- # against the same block of test/fixtures/explain/operators.json (#255). That
- # is the reachability this job was built to provide. The genre census still
- # only reports: it has no committed fixture to be asserted against.
+ # block of test/fixtures/explain/values.json, the OPERATOR IL-head census
+ # against the same block of test/fixtures/explain/operators.json (#255), and
+ # the TOKEN partition census against `test/fixtures/explain/tokens.json` →
+ # `corpus` (#289 B1). That is the reachability this job was built to provide.
+ # The genre census still only reports: it has no committed fixture to be
+ # asserted against.
#
# The other half of #260 needs no corpus and is not here: the README figures
# are a generated projection of that same fixture, gated by
- # `explain:value-census:readme:check` inside `lint:ci`.
+ # `explain:value-census:readme:check` and `explain:token-census:readme:check`
+ # inside `lint:ci`.
corpus:
name: Corpus census
needs: [checks]
@@ -218,6 +221,12 @@ jobs:
echo '```'
bun run explain:operator-census 2>&1
echo '```'
+ echo
+ echo "### Token partition census"
+ echo
+ echo '```'
+ bun run explain:token-census 2>&1
+ echo '```'
} >> "$GITHUB_STEP_SUMMARY"
# After the summary, so a drifted run still publishes its figures — the
# numbers are what tells you whether the drift is an intended emission
@@ -226,6 +235,8 @@ jobs:
run: bun run explain:value-census:check
- name: Operator census matches the committed fixture
run: bun run explain:operator-census:check
+ - name: Token partition census matches the committed fixture
+ run: bun run explain:token-census:check
chr-smoke:
name: CHR smoke (stable)
diff --git a/GLOSSARY.txt b/GLOSSARY.txt
index 13e0521..9245864 100644
--- a/GLOSSARY.txt
+++ b/GLOSSARY.txt
@@ -699,3 +699,6 @@ arities # plural of arity — the operand counts an operator accepted, which is
typeofvalue # RouterOS IL renders `[:typeof $x]` as `/typeofvalue=$x` — the directive and its argument name run together with no separator
juxt # short for juxtaposition — the unnamed ` ( a b)` node RouterOS builds for space-separated paren contents; used in ids like `any-juxt`
punct # short for punctuation — the shape class for a candidate spelling made of symbol bytes, as opposed to a word; used in the `bogus-punct` control id
+gapless # of a token partition: every byte in [0, input.bytes) belongs to exactly one token, sorted by start, no gaps (#289 B1)
+SCIP # Source Code Intelligence Protocol — the symbol-identity axis for `data.tokens[]`'s later vocabulary (#264 B5)
+unclassified # the provisional token class for a byte no analyzer claims; a first-class answer in the gapless partition (#289 B1)
diff --git a/commands/explain/README.md b/commands/explain/README.md
index 02e410c..d969177 100644
--- a/commands/explain/README.md
+++ b/commands/explain/README.md
@@ -1199,10 +1199,29 @@ And the grounded complement — asked, and refused:
| `]` | rejected |
-None of this emits a span yet. `src/explain/operators.ts` is data plus
-accessors; the operator fill for `data.tokens[]` is #264's B2 and the
-`centrs → highlight` projection is B4, and both read this table rather than
-re-deriving it.
+
+ The token census is re-derivable with `bun run explain:token-census` and
+ covers 948 source scripts. The figures below are generated from
+ `test/fixtures/explain/tokens.json` → `corpus` by
+ `bun run explain:token-census:readme` and gated against it by
+ `bun run explain:token-census:readme:check`; the fixture itself is gated
+ against a fresh corpus run by `bun run explain:token-census:check`. Of
+ 1,426,731 analyzed bytes, 391,474 are classified (27.44%), the remaining
+ 1,035,257 are `unclassified`. The census emits 46,580 tokens (avg 49.1 per
+ script). Every byte belongs to exactly one token — sorted by `start`, no
+ gaps, no overlaps, `join(slice) === input` — and the `class` field is
+ provisional until #264 B5. Each B2 fill should move the classified
+ percentage.
+
+
+B1's `data.tokens[]` is live behind `--tokens` — a total, gapless byte
+partition whose `class` is provisional until #264 B5 (every unclaimed byte is
+`unclassified`). Its only fill source today is `data.spans[]`, so an operator
+fill needs a claim seam of its own: `spans[]` is the proof-only facet (comment
+runs and resolved variable occurrences) and must not grow an operator class.
+`src/explain/operators.ts` is still data plus accessors; the operator fill for
+that partition is #264's B2 and the `centrs → highlight` projection is B4, and
+both read this table rather than re-deriving it.
### Designed, not implemented (the CLI surface, #202b)
diff --git a/docs/CLI.md b/docs/CLI.md
index 4ffb2a1..ec0006c 100644
--- a/docs/CLI.md
+++ b/docs/CLI.md
@@ -223,6 +223,7 @@ Usage: centrs explain '' [flags]
| `--complete` | | Continuation candidates at the cursor. Live evidence — offline emits a tip and enumerates nothing. |
| `--schema` | | Path enumeration (verbs, args, types, enums). Live evidence — offline emits a tip and enumerates nothing. |
| `--curl` | | Render a ready-to-edit REST curl for statements covered by a runtime-tested mapping rule. |
+| `--tokens` | | Emit the total, gapless token partition behind `data.tokens[]` (provisional `class` until #264 B5: every byte not claimed by an analyzer is `unclassified`). |
| `--format` | <text\|json\|yaml> | Output format for the CLI response. |
| `--json` | | Shortcut for `--format json`. |
| `--yaml` | | Shortcut for `--format yaml`. |
diff --git a/package.json b/package.json
index 3e7d180..85d1826 100644
--- a/package.json
+++ b/package.json
@@ -64,7 +64,7 @@
"lint": "bun run lint:biome && bun run lint:typecheck",
"lint:git:commit": "bun run lint:biome",
"lint:git:push": "bun run lint:ci",
- "lint:ci": "bun run lint && bun run docs:cli:check && bun run explain:value-census:readme:check && bun run explain:operator-readme:check && bun run lint:markdownlint && bun run lint:cspell && bun run lint:secretlint",
+ "lint:ci": "bun run lint && bun run docs:cli:check && bun run explain:value-census:readme:check && bun run explain:token-census:readme:check && bun run explain:operator-readme:check && bun run lint:markdownlint && bun run lint:cspell && bun run lint:secretlint",
"lint:cspell": "cspell lint . --gitignore",
"lint:cspell:all": "cspell lint . --no-gitignore",
"lint:markdownlint": "markdownlint-cli2 \"**/*.md\" \"!.scratch/**\"",
@@ -115,6 +115,10 @@
"explain:value-census:check": "bun run scripts/explain-value-census.ts --check",
"explain:value-census:readme": "bun run scripts/explain-value-census.ts --readme",
"explain:value-census:readme:check": "bun run scripts/explain-value-census.ts --readme --check",
+ "explain:token-census": "bun run scripts/explain-token-census.ts",
+ "explain:token-census:check": "bun run scripts/explain-token-census.ts --check",
+ "explain:token-census:readme": "bun run scripts/explain-token-census.ts --readme",
+ "explain:token-census:readme:check": "bun run scripts/explain-token-census.ts --readme --check",
"security": "bun run lint:secretlint && bun run lint:markdownlint"
},
"type": "module",
diff --git a/scripts/explain-token-census.ts b/scripts/explain-token-census.ts
new file mode 100644
index 0000000..4840b45
--- /dev/null
+++ b/scripts/explain-token-census.ts
@@ -0,0 +1,393 @@
+#!/usr/bin/env bun
+/**
+ * Token-partition census over the `explain` corpus (#264 B1).
+ *
+ * B1 emits `data.tokens[]`: a total, gapless byte partition where every byte in
+ * `[0, input.bytes)` belongs to exactly one token, sorted by `start`, with no
+ * gaps and `join(slice) === input`. Every byte no analyzer claims is
+ * `unclassified`; filling those holes is B2. The `class` field is provisional
+ * until #264 B5, and the deliverable here is a **number**: percent of corpus
+ * bytes classified (not `unclassified`), reported the way the other census
+ * figures are — a counter in a generated block, never prose (#260).
+ *
+ * That number is the progress bar for every B2 fill, and each fill PR should
+ * move it.
+ *
+ * ```
+ * bun run explain:token-census # markdown
+ * bun run explain:token-census --json # the fixture's `corpus` block
+ * bun run explain:token-census --check # gate: fresh census vs the fixture
+ * bun run explain:token-census --readme # rewrite the README block from the fixture
+ * bun run explain:token-census --readme --check # gate: README block vs the fixture
+ * ```
+ *
+ * The corpus is not in this repo (see `corpus-fetch.ts` / #186). A sibling
+ * `lsp-routeros-ts` checkout is used when present, otherwise the snapshot
+ * pinned by `bun run corpus:fetch` (source + sha256 announced on stderr).
+ */
+
+import { Database } from "bun:sqlite";
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { analyzeCoordinates } from "../src/explain/coordinates.ts";
+import { explainCommand } from "../src/explain.ts";
+import {
+ describeResolution,
+ resolveCorpusDb,
+ unreachableMessage,
+} from "./corpus-fetch.ts";
+
+export interface TokenCensus {
+ sourceScripts: number;
+ totalBytes: number;
+ classifiedBytes: number;
+ unclassifiedBytes: number;
+ classifiedPct: number;
+ totalTokens: number;
+ avgTokensPerScript: number;
+ classCounts: Record;
+ classByteCounts: Record;
+}
+
+function bump(counts: Record, key: string, by = 1): void {
+ counts[key] = (counts[key] ?? 0) + by;
+}
+
+function flag(args: readonly string[], name: string): string | undefined {
+ const at = args.indexOf(name);
+ return at < 0 ? undefined : args[at + 1];
+}
+
+export function census(scripts: readonly string[]): TokenCensus {
+ let totalBytes = 0;
+ let classifiedBytes = 0;
+ let totalTokens = 0;
+ const classCounts: Record = {};
+ const classByteCounts: Record = {};
+ const invariantFailures: string[] = [];
+
+ for (const [index, text] of scripts.entries()) {
+ const data = explainCommand(text, { tokens: true });
+ const tokensLocal = data.tokens ?? [];
+ const bytes = data.input.bytes;
+ totalBytes += bytes;
+ totalTokens += tokensLocal.length;
+
+ // A failure has to name the script that produced it. `gap/overlap at 17`
+ // on its own is unactionable in CI: the corpus is not in this repo, so
+ // without the row index and a snippet there is nothing to go look at.
+ const fail = (what: string): void => {
+ invariantFailures.push(
+ `script #${index} (${JSON.stringify(text.slice(0, 60))}${text.length > 60 ? "…" : ""}): ${what}`,
+ );
+ };
+
+ // Invariant checks (also the fact B1 must not break).
+ if (bytes === 0) {
+ if (tokensLocal.length !== 0) fail("empty input should have 0 tokens");
+ continue;
+ }
+ if (tokensLocal.length === 0) {
+ fail(`no tokens for non-empty input of ${bytes} bytes`);
+ continue;
+ }
+ if (tokensLocal[0]?.start !== 0) fail("first token not at 0");
+ if (tokensLocal[tokensLocal.length - 1]?.end !== bytes)
+ fail("last token not at bytes");
+ let hasInvalidRange = false;
+ for (const t of tokensLocal) {
+ if (
+ !Number.isInteger(t.start) ||
+ !Number.isInteger(t.end) ||
+ t.start < 0 ||
+ t.end <= t.start ||
+ t.end > bytes ||
+ t.start >= bytes
+ ) {
+ fail(`invalid token range [${t.start},${t.end}) for bytes ${bytes}`);
+ hasInvalidRange = true;
+ break;
+ }
+ }
+ if (!hasInvalidRange) {
+ for (let i = 1; i < tokensLocal.length; i++) {
+ const cur = tokensLocal[i] as { start: number; end: number };
+ const prev = tokensLocal[i - 1] as { start: number; end: number };
+ if (cur.start !== prev.end) {
+ fail(
+ `gap/overlap at token ${i}: [${prev.start},${prev.end}) then [${cur.start},${cur.end})`,
+ );
+ break;
+ }
+ }
+ const analyzed = new TextDecoder().decode(
+ analyzeCoordinates(text).analyzed,
+ );
+ const recon = tokensLocal
+ .map((t) => analyzed.slice(t.start, t.end))
+ .join("");
+ if (recon !== analyzed) fail("join(slice) !== input");
+ }
+
+ for (const t of tokensLocal) {
+ bump(classCounts, t.class);
+ bump(classByteCounts, t.class, t.end - t.start);
+ if (t.class !== "unclassified") classifiedBytes += t.end - t.start;
+ }
+ }
+
+ if (invariantFailures.length > 0) {
+ throw new Error(
+ `token partition invariant violated: ${invariantFailures.slice(0, 5).join("; ")}`,
+ );
+ }
+
+ const total = totalBytes;
+ return {
+ sourceScripts: scripts.length,
+ totalBytes,
+ classifiedBytes,
+ unclassifiedBytes: total - classifiedBytes,
+ classifiedPct: total === 0 ? 0 : (classifiedBytes / total) * 100,
+ totalTokens,
+ avgTokensPerScript: scripts.length === 0 ? 0 : totalTokens / scripts.length,
+ classCounts,
+ classByteCounts,
+ };
+}
+
+const README_PATH = join(
+ import.meta.dir,
+ "..",
+ "commands",
+ "explain",
+ "README.md",
+);
+const FIXTURE_PATH = join(
+ import.meta.dir,
+ "..",
+ "test",
+ "fixtures",
+ "explain",
+ "tokens.json",
+);
+
+const BLOCK_INDENT = " ";
+const BLOCK_BEGIN = `${BLOCK_INDENT}`;
+const BLOCK_END = `${BLOCK_INDENT}`;
+const WRAP_COLUMNS = 78;
+
+export function splitLines(text: string): string[] {
+ return text.split(/\r?\n/);
+}
+
+function lineEndingOf(text: string): string {
+ return text.includes("\r\n") ? "\r\n" : "\n";
+}
+
+function wrap(text: string): string[] {
+ const lines: string[] = [];
+ let line = BLOCK_INDENT;
+ for (const word of text.match(/(?:`[^`]*`|\S)+/g) ?? []) {
+ if (line !== BLOCK_INDENT && line.length + 1 + word.length > WRAP_COLUMNS) {
+ lines.push(line);
+ line = BLOCK_INDENT;
+ }
+ line += line === BLOCK_INDENT ? word : ` ${word}`;
+ }
+ if (line !== BLOCK_INDENT) lines.push(line);
+ return lines;
+}
+
+const count = (value: number): string => value.toLocaleString("en-US");
+
+export function renderReadmeBlock(result: TokenCensus): string[] {
+ const pct = result.classifiedPct.toFixed(2);
+ return wrap(
+ `The token census is re-derivable with \`bun run explain:token-census\` and ` +
+ `covers ${count(result.sourceScripts)} source scripts. The figures below are generated from ` +
+ "`test/fixtures/explain/tokens.json` → `corpus` by " +
+ "`bun run explain:token-census:readme` and gated against it by " +
+ "`bun run explain:token-census:readme:check`; the fixture itself is gated " +
+ "against a fresh corpus run by `bun run explain:token-census:check`. " +
+ `Of ${count(result.totalBytes)} analyzed bytes, ${count(result.classifiedBytes)} are ` +
+ `classified (${pct}%), the remaining ${count(result.unclassifiedBytes)} are \`unclassified\`. ` +
+ `The census emits ${count(result.totalTokens)} tokens ` +
+ `(avg ${result.avgTokensPerScript.toFixed(1)} per script). ` +
+ "Every byte belongs to exactly one token — sorted by `start`, no gaps, no overlaps, " +
+ "`join(slice) === input` — and the `class` field is provisional until #264 B5. " +
+ "Each B2 fill should move the classified percentage.",
+ );
+}
+
+function readFixtureCensus(): TokenCensus {
+ const fixture = JSON.parse(readFileSync(FIXTURE_PATH, "utf8")) as {
+ corpus?: TokenCensus;
+ };
+ if (fixture.corpus === undefined)
+ throw new Error(`${FIXTURE_PATH} has no \`corpus\` block`);
+ return fixture.corpus;
+}
+
+const PROVENANCE_KEYS: ReadonlySet = new Set([]);
+
+function isCountMap(value: unknown): value is Record {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function diffAgainstFixture(
+ fresh: TokenCensus,
+ pinned: TokenCensus,
+): string[] {
+ const measured = fresh as unknown as Record;
+ const committed = pinned as unknown as Record;
+ const drift: string[] = [];
+ for (const key of Object.keys(measured)) {
+ if (PROVENANCE_KEYS.has(key)) continue;
+ const a = measured[key];
+ const b = committed[key];
+ if (isCountMap(a) || isCountMap(b)) {
+ const am = isCountMap(a) ? a : {};
+ const bm = isCountMap(b) ? b : {};
+ const names = [
+ ...new Set([...Object.keys(am), ...Object.keys(bm)]),
+ ].sort();
+ for (const name of names) {
+ if (am[name] === bm[name]) continue;
+ drift.push(
+ `${key}.${name}: fixture ${bm[name] ?? "absent"}, measured ${am[name] ?? "absent"}`,
+ );
+ }
+ continue;
+ }
+ // `classifiedPct` is DERIVED from `classifiedBytes` / `totalBytes`, and
+ // both of those are compared exactly above — so real drift is already
+ // caught there. This epsilon absorbs float round-trip noise through
+ // JSON; it is not a drift allowance, and must stay far tighter than the
+ // 2 decimal places the README block renders.
+ if (key === "classifiedPct") {
+ const da = typeof a === "number" ? a : 0;
+ const db = typeof b === "number" ? b : 0;
+ if (Math.abs(da - db) < 0.0001) continue;
+ drift.push(`${key}: fixture ${b}, measured ${a}`);
+ continue;
+ }
+ if (a !== b) drift.push(`${key}: fixture ${b}, measured ${a}`);
+ }
+ for (const key of Object.keys(committed)) {
+ if (key in measured || PROVENANCE_KEYS.has(key)) continue;
+ drift.push(`${key}: in the fixture, not measured`);
+ }
+ return drift;
+}
+
+export function runReadme(check: boolean): number {
+ const rendered = renderReadmeBlock(readFixtureCensus());
+ const readme = readFileSync(README_PATH, "utf8");
+ const lines = splitLines(readme);
+ const begin = lines.indexOf(BLOCK_BEGIN);
+ const end = lines.indexOf(BLOCK_END);
+ if (begin < 0 || end < begin) {
+ console.error(
+ `::error title=explain token census::commands/explain/README.md is missing the generated token-census block markers`,
+ );
+ return 1;
+ }
+ const current = lines.slice(begin + 1, end);
+ if (current.join("\n") === rendered.join("\n")) {
+ if (!check) console.error("token-census README block already current");
+ return 0;
+ }
+ if (check) {
+ console.error(
+ "::error title=explain token census::commands/explain/README.md no longer matches " +
+ "test/fixtures/explain/tokens.json → corpus. Run `bun run explain:token-census:readme`.",
+ );
+ console.error(`--- README\n${current.join("\n")}`);
+ console.error(`+++ fixture\n${rendered.join("\n")}`);
+ return 1;
+ }
+ writeFileSync(
+ README_PATH,
+ [...lines.slice(0, begin + 1), ...rendered, ...lines.slice(end)].join(
+ lineEndingOf(readme),
+ ),
+ );
+ console.error("rewrote the token-census block in commands/explain/README.md");
+ return 0;
+}
+
+export async function main(args: readonly string[]): Promise {
+ if (args.includes("--readme")) return runReadme(args.includes("--check"));
+ const resolution = resolveCorpusDb(flag(args, "--db"));
+ if (resolution.warning) {
+ console.error(
+ `::warning title=explain token census::${resolution.warning}`,
+ );
+ }
+ if (args.includes("--check") && resolution.warning) {
+ console.error(
+ "::error title=explain token census::refusing to check against a " +
+ "corpus that is not the pinned snapshot — the result would not be " +
+ "comparable to CI's.",
+ );
+ return 1;
+ }
+ const dbPath = resolution.path;
+ if (dbPath === undefined || !(await Bun.file(dbPath).exists())) {
+ console.error(unreachableMessage("explain token census"));
+ return 1;
+ }
+ console.error(describeResolution(resolution));
+ const db = new Database(dbPath, { readonly: true });
+ let scripts: string[];
+ try {
+ scripts = (
+ db.query("SELECT text FROM source_scripts").all() as { text: string }[]
+ ).map((row) => row.text);
+ } finally {
+ db.close();
+ }
+ const result = census(scripts);
+ if (args.includes("--check")) {
+ const drift = diffAgainstFixture(result, readFixtureCensus());
+ if (drift.length > 0) {
+ console.error(
+ "::error title=explain token census::the census no longer matches " +
+ "test/fixtures/explain/tokens.json → corpus. Repin with " +
+ "`bun run explain:token-census --json`, then " +
+ "`bun run explain:token-census:readme`, and update the assertions in " +
+ "test/unit/explain-token-census.test.ts.",
+ );
+ for (const line of drift) console.error(` ${line}`);
+ return 1;
+ }
+ console.error("token census matches the committed fixture");
+ return 0;
+ }
+ const out = args.includes("--json")
+ ? JSON.stringify(result, null, 2)
+ : [
+ "| figure | value |",
+ "| ------ | ----- |",
+ ...Object.entries(result)
+ .filter(([, v]) => typeof v === "number")
+ .map(([k, v]) => `| \`${k}\` | ${v} |`),
+ "",
+ `classCounts: ${JSON.stringify(result.classCounts)}`,
+ `classByteCounts: ${JSON.stringify(result.classByteCounts)}`,
+ ].join("\n");
+ await Bun.write(Bun.stdout, `${out}\n`);
+ return 0;
+}
+
+if (import.meta.main) {
+ main(Bun.argv.slice(2))
+ .then((code) => process.exit(code))
+ .catch((error) => {
+ console.error(
+ `::error title=explain token census::${error instanceof Error ? error.message : String(error)}`,
+ );
+ process.exit(1);
+ });
+}
diff --git a/src/cli/explain.ts b/src/cli/explain.ts
index 398ec85..07cc882 100644
--- a/src/cli/explain.ts
+++ b/src/cli/explain.ts
@@ -89,6 +89,11 @@ export const explainCliCommand: CliCommandMetadata = {
description:
"Render a ready-to-edit REST curl for statements covered by a runtime-tested mapping rule.",
},
+ {
+ flag: "--tokens",
+ description:
+ "Emit the total, gapless token partition behind `data.tokens[]` (provisional `class` until #264 B5: every byte not claimed by an analyzer is `unclassified`).",
+ },
{
flag: "--format",
valueName: `<${explainOutputFormats.join("|")}>`,
@@ -107,6 +112,7 @@ export interface ExplainCliArgs {
failOn: ExplainFailOn;
facets: string[];
curl?: boolean;
+ tokens?: boolean;
format?: ExplainOutputFormat;
verbose?: boolean;
}
@@ -162,6 +168,9 @@ export function parseExplainCliArgs(args: readonly string[]): ExplainCliArgs {
case "--curl":
parsed.curl = true;
break;
+ case "--tokens":
+ parsed.tokens = true;
+ break;
case "--format": {
const value = expectValue(args, ++index, arg);
if (!(explainOutputFormats as readonly string[]).includes(value)) {
@@ -468,6 +477,7 @@ export async function runExplainCli(args: readonly string[]): Promise {
const envelope = explainEnvelope(input, {
format,
curl: parsed.curl,
+ tokens: parsed.tokens,
warnings,
tips:
parsed.facets.length > 0
diff --git a/src/explain.ts b/src/explain.ts
index 2205c10..dea832a 100644
--- a/src/explain.ts
+++ b/src/explain.ts
@@ -406,6 +406,31 @@ export interface ExplainSpan extends ExplainSpanRange {
ev: string;
}
+/**
+ * A token in the total, gapless byte partition (B1).
+ *
+ * Every byte of the analyzed input belongs to exactly one token — a
+ * contiguous, non-overlapping partition of `[0, input.bytes)` — sorted by
+ * `start`, with no gaps and `tokens.map(t => slice).join("") === input`.
+ * Byte-exact, inheriting the #215/#252 offset discipline. Offsets are on the
+ * analyzed text; `input.positionMap` applies when normalized.
+ *
+ * `class` is **explicitly provisional until #264 B5** — the vocabulary here is
+ * not the final LSP/SCIP legend, and every byte no analyzer claims becomes
+ * `unclassified`. Filling those holes is B2, one PR per fill. `unclassified`
+ * is a first-class answer, not a placeholder to be avoided.
+ */
+export type ExplainTokenClass = ExplainSpanClass | "unclassified";
+
+export interface ExplainToken extends ExplainSpanRange {
+ /**
+ * Provisional until #264 B5 — do not treat as the final vocabulary.
+ * `unclassified` means no analyzer claimed this byte.
+ */
+ class: ExplainTokenClass;
+ ev: string;
+}
+
/** The Q13 class, or `null` where offline analysis must abstain. */
export type ExplainSymbolClass = Exclude | null;
@@ -521,6 +546,17 @@ export interface ExplainData {
symbols: ExplainSymbols;
values: ExplainValues;
spans: ExplainSpan[];
+ /**
+ * Total, gapless token partition of `[0, input.bytes)`.
+ *
+ * Present only when `--tokens` is requested, matching the `--complete` /
+ * `--schema` / `--curl` facet pattern. Every byte belongs to exactly one
+ * token, sorted by `start`, with no gaps and no overlaps:
+ * `tokens.map(t => input.slice(t.start, t.end)).join("") === input`.
+ * The `class` field is provisional until #264 B5; every byte no analyzer
+ * claims is `unclassified`.
+ */
+ tokens?: ExplainToken[];
diagnostics: ExplainDiagnostic[];
evidence: ExplainEvidence[];
/**
@@ -781,6 +817,68 @@ const SPAN_CLASS_OF_SYMBOL: Record = {
parameter: "variable-parameter",
};
+/**
+ * Build the total, gapless token partition (B1).
+ *
+ * Every byte of `[0, bytes)` belongs to exactly one token: `spans` are
+ * placed sorted, no gaps become `unclassified`, and the result is sorted by
+ * `start` with no overlaps and `join(slice) === input`. The `class` field is
+ * provisional until #264 B5.
+ */
+export function buildTokens(
+ analyzed: string,
+ spans: readonly ExplainSpan[],
+): ExplainToken[] {
+ const len = analyzed.length;
+ const sorted = [...spans].sort((a, b) => a.start - b.start || a.end - b.end);
+ // Validate preconditions: spans sorted, non-overlapping, in bounds.
+ // B1 reuses existing analyzers — overlaps or out-of-bounds are a bug, not a fill.
+ let prev = 0;
+ for (const s of sorted) {
+ if (!Number.isInteger(s.start) || !Number.isInteger(s.end))
+ throw new Error(
+ `buildTokens: non-integer span [${s.start},${s.end}) for length ${len}`,
+ );
+ if (s.start < 0 || s.end <= s.start || s.end > len)
+ throw new Error(
+ `buildTokens: span out of bounds [${s.start},${s.end}) for length ${len}`,
+ );
+ if (s.start < prev)
+ throw new Error(
+ `buildTokens: overlapping spans at [${s.start},${s.end})`,
+ );
+ prev = Math.max(prev, s.end);
+ }
+ if (len === 0) return [];
+ const out: ExplainToken[] = [];
+ let cursor = 0;
+ for (const s of sorted) {
+ if (cursor < s.start) {
+ out.push({
+ start: cursor,
+ end: s.start,
+ class: "unclassified",
+ // The pass that produced an UNCLAIMED byte is the one that produced
+ // the analyzed surface it sits on — `analyzeCoordinates`, not the
+ // execute canonicalizer. A B2 fill replaces this with the pass that
+ // claimed the byte.
+ ev: EV.coordinates,
+ });
+ }
+ out.push({ start: s.start, end: s.end, class: s.class, ev: s.ev });
+ cursor = s.end;
+ }
+ if (cursor < len) {
+ out.push({
+ start: cursor,
+ end: len,
+ class: "unclassified",
+ ev: EV.coordinates, // see above
+ });
+ }
+ return out;
+}
+
function severityRank(s: ExplainSeverity): number {
return s === "error" ? 2 : s === "warning" ? 1 : 0;
}
@@ -804,6 +902,14 @@ function verdictOf(diagnostics: readonly ExplainDiagnostic[]): ExplainVerdict {
export interface ExplainCommandOptions {
/** Include a ready-to-edit REST curl on API-candidate statements. */
curl?: boolean;
+ /**
+ * Emit the total, gapless token partition behind `data.tokens[]`.
+ *
+ * Mirrors `--complete` / `--schema` / `--curl`: an opt-in facet whose
+ * presence changes the result. The `class` field is provisional until
+ * #264 B5.
+ */
+ tokens?: boolean;
}
/**
@@ -949,6 +1055,9 @@ export function explainCommand(
symbols: symbolFacts,
values: valueFacts,
spans,
+ ...(options.tokens === true
+ ? { tokens: buildTokens(analyzed, spans) }
+ : {}),
diagnostics,
evidence: citedEvidence(
structure,
@@ -1360,7 +1469,10 @@ export function explainEnvelope(
input: string,
options: ExplainEnvelopeOptions = {},
): ExplainEnvelope {
- const data = explainCommand(input, { curl: options.curl });
+ const data = explainCommand(input, {
+ curl: options.curl,
+ tokens: options.tokens,
+ });
const tips: Tip[] = [
buildTip(
"tip/explain-offline-only",
@@ -1415,6 +1527,13 @@ export interface ExplainEnvelopeOptions {
format?: ResolvedSetting;
/** Include curl rendering for API-candidate statements. */
curl?: boolean;
+ /**
+ * Include the gapless token partition behind `data.tokens[]`.
+ *
+ * Mirrors `--complete` / `--schema` / `--curl` facet pattern. Provisional
+ * vocabulary until #264 B5.
+ */
+ tokens?: boolean;
tips?: readonly Tip[];
/** Facts about the INVOCATION, not the analysis (e.g. an ignored stdin). */
warnings?: readonly Warning[];
@@ -1653,6 +1772,23 @@ function renderExplainText(
` ${span(occurrence.span).padEnd(12)} ${renderValue(occurrence)}`,
);
}
+ // `--tokens` is opt-in, so it must CHANGE this surface — a flag that only
+ // moves `--json` reads as a no-op from the default format. The header is the
+ // #289 deliverable (a coverage number) for this one input; the rows are the
+ // partition itself, `unclassified` runs included.
+ if (data.tokens !== undefined) {
+ const classified = data.tokens.reduce(
+ (sum, t) => sum + (t.class === "unclassified" ? 0 : t.end - t.start),
+ 0,
+ );
+ const bytes = data.input.bytes;
+ const pct = bytes === 0 ? 0 : (classified / bytes) * 100;
+ lines.push(
+ `tokens: ${data.tokens.length} token(s), ${classified}/${bytes} byte(s) classified (${pct.toFixed(1)}%), class provisional`,
+ );
+ for (const t of data.tokens)
+ lines.push(` ${span(t).padEnd(12)} ${t.class}`);
+ }
if (data.diagnostics.length > 0) {
lines.push("diagnostics:");
for (const d of data.diagnostics)
diff --git a/src/index.ts b/src/index.ts
index 2bcf5b1..48887db 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -463,6 +463,8 @@ export {
type ExplainSymbolClass,
type ExplainSymbolOccurrence,
type ExplainSymbols,
+ type ExplainToken,
+ type ExplainTokenClass,
type ExplainValueFacts,
type ExplainValueOccurrence,
type ExplainValueShapeFact,
diff --git a/test/fixtures/explain/tokens.json b/test/fixtures/explain/tokens.json
new file mode 100644
index 0000000..e41a533
--- /dev/null
+++ b/test/fixtures/explain/tokens.json
@@ -0,0 +1,28 @@
+{
+ "corpus": {
+ "sourceScripts": 948,
+ "totalBytes": 1426731,
+ "classifiedBytes": 391474,
+ "unclassifiedBytes": 1035257,
+ "classifiedPct": 27.438529056984112,
+ "totalTokens": 46580,
+ "avgTokensPerScript": 49.13502109704641,
+ "classCounts": {
+ "comment": 5545,
+ "unclassified": 23291,
+ "variable-global": 2769,
+ "variable-local": 11331,
+ "variable-parameter": 2242,
+ "variable-auto": 1402
+ },
+ "classByteCounts": {
+ "comment": 277806,
+ "unclassified": 1035257,
+ "variable-global": 27428,
+ "variable-local": 71695,
+ "variable-parameter": 10277,
+ "variable-auto": 4268
+ }
+ },
+ "_note": "token partition census \u2014 see scripts/explain-token-census.ts; re-derive with bun run explain:token-census --json (wrapped as {corpus: ...})"
+}
diff --git a/test/unit/explain-token-census.test.ts b/test/unit/explain-token-census.test.ts
new file mode 100644
index 0000000..4614fc8
--- /dev/null
+++ b/test/unit/explain-token-census.test.ts
@@ -0,0 +1,285 @@
+/**
+ * #289 B1 — the token partition census is re-derivable; these tests make it drift-gated.
+ *
+ * The census figures live in two places: `test/fixtures/explain/tokens.json` → `corpus`
+ * (the measurement) and the prose in `commands/explain/README.md`. The second copy
+ * is a generated projection of the first, checked here as well as in `bun run lint:ci`.
+ * Nothing here needs `corpus.sqlite`; corpus → fixture is gated by `bun run explain:token-census:check`.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import {
+ diffAgainstFixture,
+ renderReadmeBlock,
+ splitLines,
+ type TokenCensus,
+} from "../../scripts/explain-token-census.ts";
+import { analyzeCoordinates } from "../../src/explain/coordinates.ts";
+import {
+ buildTokens,
+ type ExplainSpan,
+ explainCommand,
+ explainEnvelope,
+ renderExplainEnvelope,
+} from "../../src/explain.ts";
+
+const README = readFileSync(
+ new URL("../../commands/explain/README.md", import.meta.url),
+ "utf8",
+);
+const fixture = JSON.parse(
+ readFileSync(
+ new URL("../fixtures/explain/tokens.json", import.meta.url),
+ "utf8",
+ ),
+) as { corpus: TokenCensus };
+
+const BEGIN =
+ " ";
+const END = " ";
+
+function markers(): { lines: string[]; begin: number; end: number } {
+ const lines = splitLines(README);
+ const begin = lines.indexOf(BEGIN);
+ const end = lines.indexOf(END);
+ expect(begin).toBeGreaterThanOrEqual(0);
+ expect(end).toBeGreaterThan(begin);
+ return { lines, begin, end };
+}
+
+function readmeBlock(): string[] {
+ const { lines, begin, end } = markers();
+ return lines.slice(begin + 1, end);
+}
+
+describe("#289 B1 — token partition invariant over representative inputs", () => {
+ for (const input of [
+ "",
+ '/ip/address/add address=1.2.3.4/24 interface=ether1 comment="hi"',
+ "/interface/print .proplist=name,comment",
+ ':local x 1.3; :put [:typeof $x]; /ip route print where comment~"a"',
+ "# comment only",
+ " ",
+ ":if (true) do={ :put 1 }",
+ '/system identity set name="router-🚀"',
+ "/ip address add # hello\naddress=1.1.1.1",
+ ]) {
+ test(JSON.stringify(input).slice(0, 60) || "(empty)", () => {
+ const data = explainCommand(input, { tokens: true });
+ const tokensLocal = data.tokens ?? [];
+ const bytes = data.input.bytes;
+ const analyzed = new TextDecoder().decode(
+ analyzeCoordinates(input).analyzed,
+ );
+ if (bytes === 0) {
+ expect(tokensLocal).toEqual([]);
+ return;
+ }
+ expect(tokensLocal.length).toBeGreaterThan(0);
+ expect(tokensLocal[0]?.start).toBe(0);
+ expect(tokensLocal[tokensLocal.length - 1]?.end).toBe(bytes);
+ for (let i = 1; i < tokensLocal.length; i++) {
+ expect(tokensLocal[i]?.start).toBe(tokensLocal[i - 1]?.end);
+ }
+ for (const t of tokensLocal) {
+ expect(t.start).toBeGreaterThanOrEqual(0);
+ expect(t.end).toBeLessThanOrEqual(bytes);
+ expect(t.start).toBeLessThan(t.end);
+ }
+ const recon = tokensLocal
+ .map((t) => analyzed.slice(t.start, t.end))
+ .join("");
+ expect(recon).toBe(analyzed);
+ // Every `spans` entry is a token with the same class.
+ for (const s of data.spans) {
+ const covering = tokensLocal.find(
+ (t) => t.start === s.start && t.end === s.end,
+ );
+ expect(covering).toBeDefined();
+ expect(covering?.class).toBe(s.class);
+ }
+ });
+ }
+
+ test("without --tokens the facet is absent", () => {
+ const data = explainCommand("/ip address add address=1.1.1.1");
+ expect(data.tokens).toBeUndefined();
+ });
+
+ test("`unclassified` is a first-class class, not a gap", () => {
+ // Whitespace-only input has no analyzer claim, so the partition is a
+ // single `unclassified` token — this holds regardless of how many B2
+ // fills land. Tying the assertion to a specific command (e.g.
+ // `/ip/address/add ...`) would make it fail once that command becomes
+ // fully classified.
+ const data = explainCommand(" ", {
+ tokens: true,
+ });
+ expect(data.tokens?.every((t) => typeof t.class === "string")).toBe(true);
+ // `ev` is the pass that produced the byte: an UNCLAIMED byte comes from
+ // the coordinate analysis (`e1`), not the execute canonicalizer (`e0`).
+ expect(data.tokens).toEqual([
+ { start: 0, end: 3, class: "unclassified", ev: "e1" },
+ ]);
+ // The pinned census still has unclassified bytes (the deliverable number)
+ expect(fixture.corpus.classCounts["unclassified"]).toBeGreaterThan(0);
+ });
+
+ test("normalized input: offsets on analyzed text, join(slice) === analyzed", () => {
+ const input = '/system identity set name="router-🚀"';
+ const data = explainCommand(input, { tokens: true });
+ const analyzed = new TextDecoder().decode(
+ analyzeCoordinates(input).analyzed,
+ );
+ expect(data.input.normalized).toBe(true);
+ expect(data.input.bytes).toBe(analyzed.length);
+ const recon = (data.tokens ?? [])
+ .map((t) => analyzed.slice(t.start, t.end))
+ .join("");
+ expect(recon).toBe(analyzed);
+ expect(data.tokens?.at(-1)?.end).toBe(analyzed.length);
+ });
+});
+
+describe("#289 B1 — buildTokens rejects a span set it cannot partition", () => {
+ // These guards are unreachable from `explainCommand` today (the only span
+ // producers are the comment and symbol walkers, and 40k fuzzed inputs never
+ // trip them). Reachable only by calling `buildTokens` directly — which is
+ // exactly why they need a direct test, or nothing goes red when one is
+ // deleted.
+ const span = (start: number, end: number): ExplainSpan => ({
+ start,
+ end,
+ class: "comment",
+ ev: "e2",
+ });
+
+ test("a non-integer offset", () => {
+ expect(() => buildTokens("abcdef", [span(0.5, 3)])).toThrow(
+ /non-integer span/,
+ );
+ });
+
+ test("a negative start reports bounds, not overlap", () => {
+ expect(() => buildTokens("abcdef", [span(-1, 3)])).toThrow(
+ /span out of bounds/,
+ );
+ });
+
+ test("an end past the input", () => {
+ expect(() => buildTokens("abcdef", [span(2, 99)])).toThrow(
+ /span out of bounds/,
+ );
+ });
+
+ test("an empty or reversed span", () => {
+ expect(() => buildTokens("abcdef", [span(3, 3)])).toThrow(
+ /span out of bounds/,
+ );
+ expect(() => buildTokens("abcdef", [span(5, 3)])).toThrow(
+ /span out of bounds/,
+ );
+ });
+
+ test("two spans that overlap, in either input order", () => {
+ expect(() => buildTokens("abcdef", [span(0, 4), span(2, 6)])).toThrow(
+ /overlapping spans/,
+ );
+ expect(() => buildTokens("abcdef", [span(2, 6), span(0, 4)])).toThrow(
+ /overlapping spans/,
+ );
+ });
+
+ test("unsorted but disjoint spans still partition", () => {
+ expect(buildTokens("abcdef", [span(4, 6), span(0, 2)])).toEqual([
+ { start: 0, end: 2, class: "comment", ev: "e2" },
+ { start: 2, end: 4, class: "unclassified", ev: "e1" },
+ { start: 4, end: 6, class: "comment", ev: "e2" },
+ ]);
+ });
+
+ test("empty input has no tokens", () => {
+ expect(buildTokens("", [])).toEqual([]);
+ });
+});
+
+describe("#289 B1 — `--tokens` changes the DEFAULT surface, not just `--json`", () => {
+ const input = ":local x 1 # note $x";
+
+ test("the text render gains a tokens section with the coverage number", () => {
+ const off = renderExplainEnvelope(explainEnvelope(input), "text");
+ const on = renderExplainEnvelope(
+ explainEnvelope(input, { tokens: true }),
+ "text",
+ );
+ expect(off).not.toContain("tokens:");
+ expect(on).toContain("tokens:");
+ const header = splitLines(on).find((l) => l.startsWith("tokens:"));
+ expect(header).toMatch(
+ /^tokens: \d+ token\(s\), \d+\/\d+ byte\(s\) classified \(\d+\.\d%\), class provisional$/,
+ );
+ });
+
+ test("every token is a row, `unclassified` runs included", () => {
+ const envelope = explainEnvelope(input, { tokens: true });
+ const rows = splitLines(renderExplainEnvelope(envelope, "text"));
+ const at = rows.findIndex((l) => l.startsWith("tokens:"));
+ const tokens = envelope.data.tokens ?? [];
+ expect(tokens.length).toBeGreaterThan(1);
+ expect(rows.slice(at + 1, at + 1 + tokens.length)).toEqual(
+ tokens.map((t) => ` ${`[${t.start},${t.end})`.padEnd(12)} ${t.class}`),
+ );
+ expect(tokens.some((t) => t.class === "unclassified")).toBe(true);
+ });
+});
+
+describe("#289 B1 — token-census drift gate", () => {
+ test("the README block is the fixture's `corpus` block, rendered", () => {
+ expect(readmeBlock().join("\n")).toBe(
+ renderReadmeBlock(fixture.corpus).join("\n"),
+ );
+ });
+
+ test("the block locator survives a CRLF checkout", () => {
+ const crlf = README.replace(/\r?\n/g, "\r\n");
+ const { begin, end } = markers();
+ expect(splitLines(crlf).indexOf(BEGIN)).toBe(begin);
+ expect(splitLines(crlf).indexOf(END)).toBe(end);
+ expect(crlf.split("\n").indexOf(BEGIN)).toBe(-1);
+ });
+
+ test("a rendered line never breaks a code span", () => {
+ for (const line of readmeBlock())
+ expect(line.split("`").length % 2).toBe(1);
+ });
+
+ test("the fixture comparator finds drift in a scalar and inside a count map", () => {
+ expect(diffAgainstFixture(fixture.corpus, fixture.corpus)).toEqual([]);
+ expect(
+ diffAgainstFixture(
+ {
+ ...fixture.corpus,
+ totalBytes: fixture.corpus.totalBytes + 1,
+ classCounts: { ...fixture.corpus.classCounts, unclassified: 0 },
+ },
+ fixture.corpus,
+ ).map((line) => line.split(":")[0]),
+ ).toEqual(["totalBytes", "classCounts.unclassified"]);
+ });
+
+ test("a tally is compared by entry, so key ORDER is not census data", () => {
+ const reversed = Object.fromEntries(
+ Object.entries(fixture.corpus.classCounts).reverse(),
+ );
+ expect(Object.keys(reversed)).not.toEqual(
+ Object.keys(fixture.corpus.classCounts),
+ );
+ expect(
+ diffAgainstFixture(
+ { ...fixture.corpus, classCounts: reversed },
+ fixture.corpus,
+ ),
+ ).toEqual([]);
+ });
+});