feat(explain): total, gapless token partition behind --tokens (#289 B1)

.github/workflows/ci.yaml

@@ -158,14 +158,17 @@ jobs:
   #
   # What gates here (#260): the FETCH — a hash mismatch is a hard failure, or
   # the pin is decorative — the VALUE census against the committed `corpus`
-  # block of test/fixtures/explain/values.json, and the OPERATOR IL-head census
-  # against the same block of test/fixtures/explain/operators.json (#255). That
-  # is the reachability this job was built to provide. The genre census still
-  # only reports: it has no committed fixture to be asserted against.
+  # block of test/fixtures/explain/values.json, the OPERATOR IL-head census
+  # against the same block of test/fixtures/explain/operators.json (#255), and
+  # the TOKEN partition census against `test/fixtures/explain/tokens.json` →
+  # `corpus` (#289 B1). That is the reachability this job was built to provide.
+  # The genre census still only reports: it has no committed fixture to be
+  # asserted against.
   #
   # The other half of #260 needs no corpus and is not here: the README figures
   # are a generated projection of that same fixture, gated by
-  # `explain:value-census:readme:check` inside `lint:ci`.
+  # `explain:value-census:readme:check` and `explain:token-census:readme:check`
+  # inside `lint:ci`.
   corpus:
     name: Corpus census
     needs: [checks]
@@ -218,6 +221,12 @@ jobs:
             echo '```'
             bun run explain:operator-census 2>&1
             echo '```'
+            echo
+            echo "### Token partition census"
+            echo
+            echo '```'
+            bun run explain:token-census 2>&1
+            echo '```'
           } >> "$GITHUB_STEP_SUMMARY"
       # After the summary, so a drifted run still publishes its figures — the
       # numbers are what tells you whether the drift is an intended emission
@@ -226,6 +235,8 @@ jobs:
         run: bun run explain:value-census:check
       - name: Operator census matches the committed fixture
         run: bun run explain:operator-census:check
+      - name: Token partition census matches the committed fixture
+        run: bun run explain:token-census:check
 
   chr-smoke:
     name: CHR smoke (stable)

GLOSSARY.txt

@@ -699,3 +699,6 @@ arities # plural of arity — the operand counts an operator accepted, which is
 typeofvalue # RouterOS IL renders `[:typeof $x]` as `/typeofvalue=$x` — the directive and its argument name run together with no separator
 juxt # short for juxtaposition — the unnamed ` (  a b)` node RouterOS builds for space-separated paren contents; used in ids like `any-juxt`
 punct # short for punctuation — the shape class for a candidate spelling made of symbol bytes, as opposed to a word; used in the `bogus-punct` control id
+gapless # of a token partition: every byte in [0, input.bytes) belongs to exactly one token, sorted by start, no gaps (#289 B1)
+SCIP # Source Code Intelligence Protocol — the symbol-identity axis for `data.tokens[]`'s later vocabulary (#264 B5)
+unclassified # the provisional token class for a byte no analyzer claims; a first-class answer in the gapless partition (#289 B1)

commands/explain/README.md

@@ -1199,10 +1199,29 @@ And the grounded complement — asked, and refused:
 | `]` | rejected |
 <!-- END GENERATED operator-table -->
 
-None of this emits a span yet. `src/explain/operators.ts` is data plus
-accessors; the operator fill for `data.tokens[]` is #264's B2 and the
-`centrs → highlight` projection is B4, and both read this table rather than
-re-deriving it.
+  <!-- BEGIN GENERATED token-census — regenerate with `bun run explain:token-census:readme` -->
+  The token census is re-derivable with `bun run explain:token-census` and
+  covers 948 source scripts. The figures below are generated from
+  `test/fixtures/explain/tokens.json` → `corpus` by
+  `bun run explain:token-census:readme` and gated against it by
+  `bun run explain:token-census:readme:check`; the fixture itself is gated
+  against a fresh corpus run by `bun run explain:token-census:check`. Of
+  1,426,731 analyzed bytes, 391,474 are classified (27.44%), the remaining
+  1,035,257 are `unclassified`. The census emits 46,580 tokens (avg 49.1 per
+  script). Every byte belongs to exactly one token — sorted by `start`, no
+  gaps, no overlaps, `join(slice) === input` — and the `class` field is
+  provisional until #264 B5. Each B2 fill should move the classified
+  percentage.
+  <!-- END GENERATED token-census -->
+
+B1's `data.tokens[]` is live behind `--tokens` — a total, gapless byte
+partition whose `class` is provisional until #264 B5 (every unclaimed byte is
+`unclassified`). Its only fill source today is `data.spans[]`, so an operator
+fill needs a claim seam of its own: `spans[]` is the proof-only facet (comment
+runs and resolved variable occurrences) and must not grow an operator class.
+`src/explain/operators.ts` is still data plus accessors; the operator fill for
+that partition is #264's B2 and the `centrs → highlight` projection is B4, and
+both read this table rather than re-deriving it.
 
 ### Designed, not implemented (the CLI surface, #202b)
 

docs/CLI.md

@@ -223,6 +223,7 @@ Usage: centrs explain '<input>' [flags]
 | `--complete` |  | Continuation candidates at the cursor. Live evidence — offline emits a tip and enumerates nothing. |
 | `--schema` |  | Path enumeration (verbs, args, types, enums). Live evidence — offline emits a tip and enumerates nothing. |
 | `--curl` |  | Render a ready-to-edit REST curl for statements covered by a runtime-tested mapping rule. |
+| `--tokens` |  | Emit the total, gapless token partition behind `data.tokens[]` (provisional `class` until #264 B5: every byte not claimed by an analyzer is `unclassified`). |
 | `--format` | &lt;text\|json\|yaml&gt; | Output format for the CLI response. |
 | `--json` |  | Shortcut for `--format json`. |
 | `--yaml` |  | Shortcut for `--format yaml`. |

package.json

@@ -64,7 +64,7 @@
 		"lint": "bun run lint:biome && bun run lint:typecheck",
 		"lint:git:commit": "bun run lint:biome",
 		"lint:git:push": "bun run lint:ci",
-		"lint:ci": "bun run lint && bun run docs:cli:check && bun run explain:value-census:readme:check && bun run explain:operator-readme:check && bun run lint:markdownlint && bun run lint:cspell && bun run lint:secretlint",
+		"lint:ci": "bun run lint && bun run docs:cli:check && bun run explain:value-census:readme:check && bun run explain:token-census:readme:check && bun run explain:operator-readme:check && bun run lint:markdownlint && bun run lint:cspell && bun run lint:secretlint",
 		"lint:cspell": "cspell lint . --gitignore",
 		"lint:cspell:all": "cspell lint . --no-gitignore",
 		"lint:markdownlint": "markdownlint-cli2 \"**/*.md\" \"!.scratch/**\"",
@@ -115,6 +115,10 @@
 		"explain:value-census:check": "bun run scripts/explain-value-census.ts --check",
 		"explain:value-census:readme": "bun run scripts/explain-value-census.ts --readme",
 		"explain:value-census:readme:check": "bun run scripts/explain-value-census.ts --readme --check",
+		"explain:token-census": "bun run scripts/explain-token-census.ts",
+		"explain:token-census:check": "bun run scripts/explain-token-census.ts --check",
+		"explain:token-census:readme": "bun run scripts/explain-token-census.ts --readme",
+		"explain:token-census:readme:check": "bun run scripts/explain-token-census.ts --readme --check",
 		"security": "bun run lint:secretlint && bun run lint:markdownlint"
 	},
 	"type": "module",