feat(explain): operator fill on the token partition seam (#290 B2)

GLOSSARY.txt

@@ -702,3 +702,8 @@ punct # short for punctuation — the shape class for a candidate spelling made
 gapless # of a token partition: every byte in [0, input.bytes) belongs to exactly one token, sorted by start, no gaps (#289 B1)
 SCIP # Source Code Intelligence Protocol — the symbol-identity axis for `data.tokens[]`'s later vocabulary (#264 B5)
 unclassified # the provisional token class for a byte no analyzer claims; a first-class answer in the gapless partition (#289 B1)
+
+# #290 B2 operator-fill test vocabulary
+xand
+andx
+alnum

commands/explain/README.md

@@ -1206,8 +1206,8 @@ And the grounded complement — asked, and refused:
   `bun run explain:token-census:readme` and gated against it by
   `bun run explain:token-census:readme:check`; the fixture itself is gated
   against a fresh corpus run by `bun run explain:token-census:check`. Of
-  1,426,731 analyzed bytes, 391,474 are classified (27.44%), the remaining
-  1,035,257 are `unclassified`. The census emits 46,580 tokens (avg 49.1 per
+  1,426,731 analyzed bytes, 408,307 are classified (28.62%), the remaining
+  1,018,424 are `unclassified`. The census emits 72,264 tokens (avg 76.2 per
   script). Every byte belongs to exactly one token — sorted by `start`, no
   gaps, no overlaps, `join(slice) === input` — and the `class` field is
   provisional until #264 B5. Each B2 fill should move the classified
@@ -1216,12 +1216,11 @@ And the grounded complement — asked, and refused:
 
 B1's `data.tokens[]` is live behind `--tokens` — a total, gapless byte
 partition whose `class` is provisional until #264 B5 (every unclaimed byte is
-`unclassified`). Its only fill source today is `data.spans[]`, so an operator
-fill needs a claim seam of its own: `spans[]` is the proof-only facet (comment
-runs and resolved variable occurrences) and must not grow an operator class.
-`src/explain/operators.ts` is still data plus accessors; the operator fill for
-that partition is #264's B2 and the `centrs → highlight` projection is B4, and
-both read this table rather than re-deriving it.
+`unclassified`). Since B1 its only fill source was `data.spans[]` (comment runs
+and resolved variable occurrences); **`#290`'s operator fill is the first B2
+fill** — `src/explain/operator-tokens.ts` claims `operator` bytes on the
+residual left by spans, with fill order enforced by argument order to
+`buildTokens` ([#290 design decision 1](../..//../src/explain.ts)). `ExplainTokenClass` is `ExplainSpanClass | "operator" | "unclassified"` — one provisional `operator` class for all 26 spellings + 2 aliases, not per-operator or per-category (`#264` B5). `ExplainSpanClass` and `data.spans[]` stay proof-only; `src/explain/operators.ts` remains data plus accessors, and the operator table above is its source. The `centrs → highlight` projection is B4 and reads both.
 
 ### Designed, not implemented (the CLI surface, #202b)
 

src/explain.ts

@@ -105,6 +105,7 @@ import {
 	type DefectCode,
 	isPositionalFact,
 } from "./explain/defects.ts";
+import { operatorSpans } from "./explain/operator-tokens.ts";
 import { type Resolution, resolveDocument } from "./explain/pathresolve.ts";
 import { collectStringEscapeDefects } from "./explain/quoted-string.ts";
 import { segmentStatements } from "./explain/segment.ts";
@@ -406,6 +407,21 @@ export interface ExplainSpan extends ExplainSpanRange {
 	ev: string;
 }
 
+/**
+ * One analyzer's contribution to the token partition.
+ *
+ * A fill is already-sorted, non-overlapping spans in document (analyzed-byte)
+ * space. `buildTokens` takes fills **in order** — fill 0 claims first, fill 1
+ * sees only the residual, and so on — so the argument order IS the fill order
+ * (#290 design decision 1) and a structural ambiguity (`/` path sep vs `division`)
+ * resolves by which analyzer came first rather than by a smarter byte scanner.
+ *
+ * A fill is typed as `ExplainToken[]` because B2 fills (operator, then
+ * path/arg) introduce classes outside `ExplainSpanClass`; the proof-only
+ * `spans[]` (ExplainSpan[]) remains a subtype and so fits the same slot.
+ */
+export type TokenFill = readonly ExplainToken[];
+
 /**
  * A token in the total, gapless byte partition (B1).
  *
@@ -420,7 +436,7 @@ export interface ExplainSpan extends ExplainSpanRange {
  * `unclassified`. Filling those holes is B2, one PR per fill. `unclassified`
  * is a first-class answer, not a placeholder to be avoided.
  */
-export type ExplainTokenClass = ExplainSpanClass | "unclassified";
+export type ExplainTokenClass = ExplainSpanClass | "operator" | "unclassified";
 
 export interface ExplainToken extends ExplainSpanRange {
 	/**
@@ -600,6 +616,7 @@ const EV = {
 	symbols: "e7",
 	transport: "e8",
 	values: "e9",
+	operators: "e10",
 } as const;
 
 type EvidenceKey = keyof typeof EV;
@@ -675,6 +692,13 @@ const EVIDENCE: Record<EvidenceKey, ExplainEvidence> = {
 		basis: "heuristic",
 		outcome: "ok",
 	},
+	operators: {
+		id: EV.operators,
+		source: "canonicalizer",
+		probe: "operatorSpans",
+		basis: "heuristic",
+		outcome: "ok",
+	},
 };
 
 /** Diagnostic rendering for each defect class. */
@@ -817,32 +841,94 @@ const SPAN_CLASS_OF_SYMBOL: Record<string, ExplainSpanClass> = {
 	parameter: "variable-parameter",
 };
 
+/**
+ * Complement of claimed spans within `[0, len)` — the residual runs a fill
+ * sees.
+ *
+ * Factored out of `buildTokens` so every fill can compute it and so tests can
+ * assert it directly. `claimed` need not be sorted; the result is sorted and
+ * coalesced with no gaps or overlaps relative to the claimed set.
+ */
+export function residualRanges(
+	len: number,
+	claimed: readonly ExplainSpanRange[],
+): { start: number; end: number }[] {
+	if (len === 0) return [];
+	const sorted = [...claimed].sort(
+		(a, b) => a.start - b.start || a.end - b.end,
+	);
+	let cursor = 0;
+	const out: { start: number; end: number }[] = [];
+	for (const r of sorted) {
+		if (cursor < r.start) out.push({ start: cursor, end: r.start });
+		cursor = Math.max(cursor, r.end);
+	}
+	if (cursor < len) out.push({ start: cursor, end: len });
+	return out;
+}
+
 /**
  * Build the total, gapless token partition (B1).
  *
  * Every byte of `[0, bytes)` belongs to exactly one token: `spans` are
  * placed sorted, no gaps become `unclassified`, and the result is sorted by
  * `start` with no overlaps and `join(slice) === input`. The `class` field is
  * provisional until #264 B5.
+ *
+ * B2: `buildTokens` now takes an **ordered list of fills**. Fill 0 claims
+ * first, fill 1 sees only the residual of fill 0, and so on — the argument
+ * order IS the fill order (#290 design decision 1). An overlap across fills is
+ * a hard throw (structural impossibility is achieved by callers offering only
+ * residual; the throw is the safety net). For backward compatibility a single
+ * fill may be passed as a flat span array.
  */
 export function buildTokens(
 	analyzed: string,
-	spans: readonly ExplainSpan[],
+	spansOrFills: readonly ExplainSpan[] | readonly TokenFill[],
 ): ExplainToken[] {
 	const len = analyzed.length;
-	const sorted = [...spans].sort((a, b) => a.start - b.start || a.end - b.end);
-	// Validate preconditions: spans sorted, non-overlapping, in bounds.
-	// B1 reuses existing analyzers — overlaps or out-of-bounds are a bug, not a fill.
+	// Normalize the overload: a flat span array is one fill.
+	let fills: readonly TokenFill[];
+	if (spansOrFills.length === 0) {
+		fills = [];
+	} else {
+		const first = spansOrFills[0] as unknown as Record<string, unknown>;
+		const isFlatSpan =
+			first !== null &&
+			typeof first === "object" &&
+			"class" in first &&
+			"ev" in first &&
+			"start" in first;
+		fills = isFlatSpan
+			? ([spansOrFills] as unknown as readonly TokenFill[])
+			: (spansOrFills as readonly TokenFill[]);
+	}
+	// Validate each fill internally (non-integer, bounds, overlap within fill).
+	for (const fill of fills) {
+		const sorted = [...fill].sort((a, b) => a.start - b.start || a.end - b.end);
+		let prev = 0;
+		for (const s of sorted) {
+			if (!Number.isInteger(s.start) || !Number.isInteger(s.end))
+				throw new Error(
+					`buildTokens: non-integer span [${s.start},${s.end}) for length ${len}`,
+				);
+			if (s.start < 0 || s.end <= s.start || s.end > len)
+				throw new Error(
+					`buildTokens: span out of bounds [${s.start},${s.end}) for length ${len}`,
+				);
+			if (s.start < prev)
+				throw new Error(
+					`buildTokens: overlapping spans at [${s.start},${s.end})`,
+				);
+			prev = Math.max(prev, s.end);
+		}
+	}
+	const flat: ExplainToken[] = fills.flatMap((fill) => [...fill]);
+	const sorted = [...flat].sort((a, b) => a.start - b.start || a.end - b.end);
+	// Cross-fill overlap is also a hard throw — callers achieve impossibility by
+	// offering only residual; this is the safety net.
 	let prev = 0;
 	for (const s of sorted) {
-		if (!Number.isInteger(s.start) || !Number.isInteger(s.end))
-			throw new Error(
-				`buildTokens: non-integer span [${s.start},${s.end}) for length ${len}`,
-			);
-		if (s.start < 0 || s.end <= s.start || s.end > len)
-			throw new Error(
-				`buildTokens: span out of bounds [${s.start},${s.end}) for length ${len}`,
-			);
 		if (s.start < prev)
 			throw new Error(
 				`buildTokens: overlapping spans at [${s.start},${s.end})`,
@@ -1035,6 +1121,20 @@ export function explainCommand(
 		ev: EV.write,
 	};
 
+	// B2 fill order — the argument order IS the order (#290 design decision 1).
+	// `spans` (proof-only: comment + variables) claims first; every later fill
+	// sees only the residual left by the fills before it, so a structural
+	// ambiguity (`/` path vs division, `,` arg sep vs concat) resolves by which
+	// analyzer came first. `operatorSpans` is the first such fill.
+	const residual0 = residualRanges(analyzed.length, spans);
+	const opSpans = operatorSpans(analyzed, residual0);
+	// Future B2 fills insert here, each against the residual of prior fills:
+	// const residual1 = residualRanges(analyzed.length, [...spans, ...opSpans]);
+	// const pathSpans = pathSpansOnResidual(analyzed, residual1, ...);
+	const fills: TokenFill[] = [spans, opSpans];
+	const tokens =
+		options.tokens === true ? buildTokens(analyzed, fills) : undefined;
+
 	return {
 		input: {
 			bytes: coordinates.analyzed.length,
@@ -1055,16 +1155,15 @@ export function explainCommand(
 		symbols: symbolFacts,
 		values: valueFacts,
 		spans,
-		...(options.tokens === true
-			? { tokens: buildTokens(analyzed, spans) }
-			: {}),
+		...(tokens === undefined ? {} : { tokens }),
 		diagnostics,
 		evidence: citedEvidence(
 			structure,
 			diagnostics,
 			spans,
 			symbolFacts,
 			valueFacts,
+			tokens,
 		),
 		runtimeAcceptance: "not-proven",
 	};
@@ -1431,6 +1530,7 @@ function citedEvidence(
 	spans: readonly ExplainSpan[],
 	symbols: ExplainSymbols,
 	values: ExplainValues,
+	tokens?: readonly ExplainToken[] | readonly ExplainSpan[] | undefined,
 ): ExplainEvidence[] {
 	// `canonical` and `input` carry no `ev` of their own — they are whole-result
 	// fields, not entries in a list — so their two passes are seeded here.
@@ -1452,6 +1552,7 @@ function citedEvidence(
 		if (occurrence.facts.schemaType !== undefined)
 			cited.add(occurrence.facts.schemaType.ev);
 	}
+	if (tokens !== undefined) for (const t of tokens) cited.add(t.ev);
 	return Object.values(EVIDENCE)
 		.filter((e) => cited.has(e.id))
 		.sort((a, b) => a.id.localeCompare(b.id));