diff --git a/GLOSSARY.txt b/GLOSSARY.txt index 9245864..f7b573b 100644 --- a/GLOSSARY.txt +++ b/GLOSSARY.txt @@ -702,3 +702,10 @@ punct # short for punctuation — the shape class for a candidate spelling made gapless # of a token partition: every byte in [0, input.bytes) belongs to exactly one token, sorted by start, no gaps (#289 B1) SCIP # Source Code Intelligence Protocol — the symbol-identity axis for `data.tokens[]`'s later vocabulary (#264 B5) unclassified # the provisional token class for a byte no analyzer claims; a first-class answer in the gapless partition (#289 B1) + +# #290 B2 operator-fill test vocabulary +xand +andx +alnum +wpa # `security.authentication-types=wpa2-psk` — a wifi argument name whose interior dot the operator fill must not read as concatenation +psk # pre-shared key — the value half of the same grounded `wpa2-psk` example diff --git a/commands/explain/README.md b/commands/explain/README.md index d969177..37ff656 100644 --- a/commands/explain/README.md +++ b/commands/explain/README.md @@ -1206,8 +1206,8 @@ And the grounded complement — asked, and refused: `bun run explain:token-census:readme` and gated against it by `bun run explain:token-census:readme:check`; the fixture itself is gated against a fresh corpus run by `bun run explain:token-census:check`. Of - 1,426,731 analyzed bytes, 391,474 are classified (27.44%), the remaining - 1,035,257 are `unclassified`. The census emits 46,580 tokens (avg 49.1 per + 1,426,731 analyzed bytes, 403,516 are classified (28.28%), the remaining + 1,023,215 are `unclassified`. The census emits 63,416 tokens (avg 66.9 per script). Every byte belongs to exactly one token — sorted by `start`, no gaps, no overlaps, `join(slice) === input` — and the `class` field is provisional until #264 B5. Each B2 fill should move the classified @@ -1216,12 +1216,34 @@ And the grounded complement — asked, and refused: B1's `data.tokens[]` is live behind `--tokens` — a total, gapless byte partition whose `class` is provisional until #264 B5 (every unclaimed byte is -`unclassified`). Its only fill source today is `data.spans[]`, so an operator -fill needs a claim seam of its own: `spans[]` is the proof-only facet (comment -runs and resolved variable occurrences) and must not grow an operator class. -`src/explain/operators.ts` is still data plus accessors; the operator fill for -that partition is #264's B2 and the `centrs → highlight` projection is B4, and -both read this table rather than re-deriving it. +`unclassified`). Since B1 its only fill source was `data.spans[]` (comment runs +and resolved variable occurrences); **`#290`'s operator fill is the first B2 +fill** — `src/explain/operator-tokens.ts` claims `operator` bytes on the +residual left by spans, with fill order enforced by argument order to +`buildTokens` ([#290 design decision 1](../../src/explain.ts)). +`ExplainTokenClass` is `ExplainSpanClass | "operator" | "unclassified"` — one +provisional `operator` class for all 26 spellings + 2 aliases, not per-operator +or per-category (`#264` B5). `ExplainSpanClass` and `data.spans[]` stay +proof-only; `src/explain/operators.ts` remains data plus accessors, and the +operator table above is its source. The `centrs → highlight` projection is B4 +and reads both. + +**Where the operator fill abstains.** Fill order is the resolution mechanism, so +a byte that is structurally part of a path or an argument is left +`unclassified` for the path/arg fills rather than claimed. Three abstentions, +each grounded on the corpus device oracle (`parseil_results.il_text`): + +| Abstention | Grounding | +| ---------- | --------- | +| `, / = -` outside a `( )` group. `(` opens an expression; `[` opens a **command substitution** and `{` a block or array literal. | The IL for `[ /system/identity/get value-name=name ]` is `(evl /system/identity/get value-name=name)` — path separators and an argument separator, no division and no comparison node. | +| A spelling glued immediately after an argument `=`. | `in-interface-list=!LAN`, `.id=*2`, `oid=.1.3.6.1.2.1` — the byte after an argument `=` starts the value. | +| A spelling glued into an argument **name**. | `:foreach x in=$list` is `/foreach counter=$x` with no `(in …)` node anywhere; the IL keeps `security.authentication-types=wpa2-psk` as one name and renders `.id` as the single symbol `$.id`, never `(. …)`. | + +The first abstention has a measured cost: `find where name="x"` inside `[ … ]` +*does* lower to a real `(= $name x)` node, so 224 of the corpus's 1,259 +bracket-interior `=` bytes are genuine comparisons that now stay +`unclassified` — against 1,035 that were plain `arg=value`. A `where`-aware +fill can take them back later; claiming all 1,259 would be 82% wrong. ### Designed, not implemented (the CLI surface, #202b) diff --git a/src/explain.ts b/src/explain.ts index dea832a..1d1d8f4 100644 --- a/src/explain.ts +++ b/src/explain.ts @@ -105,6 +105,7 @@ import { type DefectCode, isPositionalFact, } from "./explain/defects.ts"; +import { operatorSpans } from "./explain/operator-tokens.ts"; import { type Resolution, resolveDocument } from "./explain/pathresolve.ts"; import { collectStringEscapeDefects } from "./explain/quoted-string.ts"; import { segmentStatements } from "./explain/segment.ts"; @@ -406,6 +407,21 @@ export interface ExplainSpan extends ExplainSpanRange { ev: string; } +/** + * One analyzer's contribution to the token partition. + * + * A fill is already-sorted, non-overlapping spans in document (analyzed-byte) + * space. `buildTokens` takes fills **in order** — fill 0 claims first, fill 1 + * sees only the residual, and so on — so the argument order IS the fill order + * (#290 design decision 1) and a structural ambiguity (`/` path sep vs `division`) + * resolves by which analyzer came first rather than by a smarter byte scanner. + * + * A fill is typed as `ExplainToken[]` because B2 fills (operator, then + * path/arg) introduce classes outside `ExplainSpanClass`; the proof-only + * `spans[]` (ExplainSpan[]) remains a subtype and so fits the same slot. + */ +export type TokenFill = readonly ExplainToken[]; + /** * A token in the total, gapless byte partition (B1). * @@ -420,7 +436,7 @@ export interface ExplainSpan extends ExplainSpanRange { * `unclassified`. Filling those holes is B2, one PR per fill. `unclassified` * is a first-class answer, not a placeholder to be avoided. */ -export type ExplainTokenClass = ExplainSpanClass | "unclassified"; +export type ExplainTokenClass = ExplainSpanClass | "operator" | "unclassified"; export interface ExplainToken extends ExplainSpanRange { /** @@ -600,6 +616,7 @@ const EV = { symbols: "e7", transport: "e8", values: "e9", + operators: "e10", } as const; type EvidenceKey = keyof typeof EV; @@ -675,6 +692,13 @@ const EVIDENCE: Record = { basis: "heuristic", outcome: "ok", }, + operators: { + id: EV.operators, + source: "canonicalizer", + probe: "operatorSpans", + basis: "heuristic", + outcome: "ok", + }, }; /** Diagnostic rendering for each defect class. */ @@ -817,6 +841,32 @@ const SPAN_CLASS_OF_SYMBOL: Record = { parameter: "variable-parameter", }; +/** + * Complement of claimed spans within `[0, len)` — the residual runs a fill + * sees. + * + * Factored out of `buildTokens` so every fill can compute it and so tests can + * assert it directly. `claimed` need not be sorted; the result is sorted and + * coalesced with no gaps or overlaps relative to the claimed set. + */ +export function residualRanges( + len: number, + claimed: readonly ExplainSpanRange[], +): { start: number; end: number }[] { + if (len === 0) return []; + const sorted = [...claimed].sort( + (a, b) => a.start - b.start || a.end - b.end, + ); + let cursor = 0; + const out: { start: number; end: number }[] = []; + for (const r of sorted) { + if (cursor < r.start) out.push({ start: cursor, end: r.start }); + cursor = Math.max(cursor, r.end); + } + if (cursor < len) out.push({ start: cursor, end: len }); + return out; +} + /** * Build the total, gapless token partition (B1). * @@ -824,25 +874,61 @@ const SPAN_CLASS_OF_SYMBOL: Record = { * placed sorted, no gaps become `unclassified`, and the result is sorted by * `start` with no overlaps and `join(slice) === input`. The `class` field is * provisional until #264 B5. + * + * B2: `buildTokens` now takes an **ordered list of fills**. Fill 0 claims + * first, fill 1 sees only the residual of fill 0, and so on — the argument + * order IS the fill order (#290 design decision 1). An overlap across fills is + * a hard throw (structural impossibility is achieved by callers offering only + * residual; the throw is the safety net). For backward compatibility a single + * fill may be passed as a flat span array. */ export function buildTokens( analyzed: string, - spans: readonly ExplainSpan[], + spansOrFills: readonly ExplainSpan[] | readonly TokenFill[], ): ExplainToken[] { const len = analyzed.length; - const sorted = [...spans].sort((a, b) => a.start - b.start || a.end - b.end); - // Validate preconditions: spans sorted, non-overlapping, in bounds. - // B1 reuses existing analyzers — overlaps or out-of-bounds are a bug, not a fill. + // Normalize the overload: a flat span array is one fill. + let fills: readonly TokenFill[]; + if (spansOrFills.length === 0) { + fills = []; + } else { + const first = spansOrFills[0] as unknown as Record; + const isFlatSpan = + first !== null && + typeof first === "object" && + "class" in first && + "ev" in first && + "start" in first; + fills = isFlatSpan + ? ([spansOrFills] as unknown as readonly TokenFill[]) + : (spansOrFills as readonly TokenFill[]); + } + // Validate each fill internally (non-integer, bounds, overlap within fill). + for (const fill of fills) { + const sorted = [...fill].sort((a, b) => a.start - b.start || a.end - b.end); + let prev = 0; + for (const s of sorted) { + if (!Number.isInteger(s.start) || !Number.isInteger(s.end)) + throw new Error( + `buildTokens: non-integer span [${s.start},${s.end}) for length ${len}`, + ); + if (s.start < 0 || s.end <= s.start || s.end > len) + throw new Error( + `buildTokens: span out of bounds [${s.start},${s.end}) for length ${len}`, + ); + if (s.start < prev) + throw new Error( + `buildTokens: overlapping spans at [${s.start},${s.end})`, + ); + prev = Math.max(prev, s.end); + } + } + const flat: ExplainToken[] = fills.flatMap((fill) => [...fill]); + const sorted = [...flat].sort((a, b) => a.start - b.start || a.end - b.end); + // Cross-fill overlap is also a hard throw — callers achieve impossibility by + // offering only residual; this is the safety net. let prev = 0; for (const s of sorted) { - if (!Number.isInteger(s.start) || !Number.isInteger(s.end)) - throw new Error( - `buildTokens: non-integer span [${s.start},${s.end}) for length ${len}`, - ); - if (s.start < 0 || s.end <= s.start || s.end > len) - throw new Error( - `buildTokens: span out of bounds [${s.start},${s.end}) for length ${len}`, - ); if (s.start < prev) throw new Error( `buildTokens: overlapping spans at [${s.start},${s.end})`, @@ -1035,6 +1121,36 @@ export function explainCommand( ev: EV.write, }; + // B2 fill order — the argument order IS the order (#290 design decision 1). + // `spans` (proof-only: comment + variables) claims first; every later fill + // sees only the residual left by the fills before it, so a structural + // ambiguity (`/` path vs division, `,` arg sep vs concat) resolves by which + // analyzer came first. `operatorSpans` is the first such fill. + // + // **The path and arg fills belong BEFORE `operatorSpans`, not after.** The + // intended end state is that `pathresolve.ts` has already claimed the `/` + // and `args.ts` the `=` by the time the operator scanner runs, so it only + // ever sees bytes nobody else wanted. Until those fills exist, the operator + // fill buys the same safety by abstaining wherever a byte is structurally + // path or argument (see `operator-tokens.ts` — three grounded abstentions). + // Inserting a fill after it would leave those abstentions doing work they + // should not have to do, and the scanner can relax them only once the fill + // that owns those bytes runs first: + // const pathSpans = pathSpansOnResidual(analyzed, residual0, ...); + // const residual1 = residualRanges(analyzed.length, [...spans, ...pathSpans]); + // const opSpans = operatorSpans(analyzed, residual1); + // const fills: TokenFill[] = [spans, pathSpans, opSpans]; + // + // Gate the scan on `options.tokens` — no residual work when the caller + // did not ask for `data.tokens`. Future B2 fills belong inside this branch. + let tokens: ExplainToken[] | undefined; + if (options.tokens === true) { + const residual0 = residualRanges(analyzed.length, spans); + const opSpans = operatorSpans(analyzed, residual0); + const fills: TokenFill[] = [spans, opSpans]; + tokens = buildTokens(analyzed, fills); + } + return { input: { bytes: coordinates.analyzed.length, @@ -1055,9 +1171,7 @@ export function explainCommand( symbols: symbolFacts, values: valueFacts, spans, - ...(options.tokens === true - ? { tokens: buildTokens(analyzed, spans) } - : {}), + ...(tokens === undefined ? {} : { tokens }), diagnostics, evidence: citedEvidence( structure, @@ -1065,6 +1179,7 @@ export function explainCommand( spans, symbolFacts, valueFacts, + tokens, ), runtimeAcceptance: "not-proven", }; @@ -1431,6 +1546,7 @@ function citedEvidence( spans: readonly ExplainSpan[], symbols: ExplainSymbols, values: ExplainValues, + tokens?: readonly ExplainToken[] | readonly ExplainSpan[] | undefined, ): ExplainEvidence[] { // `canonical` and `input` carry no `ev` of their own — they are whole-result // fields, not entries in a list — so their two passes are seeded here. @@ -1452,6 +1568,7 @@ function citedEvidence( if (occurrence.facts.schemaType !== undefined) cited.add(occurrence.facts.schemaType.ev); } + if (tokens !== undefined) for (const t of tokens) cited.add(t.ev); return Object.values(EVIDENCE) .filter((e) => cited.has(e.id)) .sort((a, b) => a.id.localeCompare(b.id)); diff --git a/src/explain/operator-tokens.ts b/src/explain/operator-tokens.ts new file mode 100644 index 0000000..9272db5 --- /dev/null +++ b/src/explain/operator-tokens.ts @@ -0,0 +1,310 @@ +/** + * B2 operator fill — claims operator bytes on the residual. + * + * This is the first B2 fill of #264. It runs **after** the proof-only spans + * (`comment` + `variable-*`) and sees only the residual left by them — fill + * order is structural, not lexical (#290 design decision 1). `/` that is a path + * separator, `,` that is an argument separator, `=` that is a name separator + * and `-` that is a hyphen in a name are resolved by who claimed the byte + * first, not by a smarter operator scanner. + * + * Two-hop census note: operator's abstention leaves some bytes `unclassified` + * that a future path/arg fill will claim. That means the classified percentage + * jumps twice for those bytes (now `unclassified`→stays, later + * `unclassified`→`path`/`arg`), which is correct — don't misread the later diff + * as "operator stole bytes". + * + * ## Abstention is the same rule three times + * + * A byte that is structurally part of a **path** or an **argument** belongs to + * a later fill, so this fill leaves it alone. Grounded on the corpus device + * oracle (`parseil_results.il_text`, the IL RouterOS actually parsed to): + * + * 1. **`[` is command substitution, not expression grouping.** `(` opens an + * expression; `[` opens a *command* and `{` a block/array literal. The IL + * for `[ /system/identity/get value-name=name ]` is + * `(evl /system/identity/get value-name=name)` — the `/` are path + * separators and the `=` is an argument separator, with no division and no + * comparison node. So the `, / = -` conservatism holds everywhere except + * directly inside `(`. Measured cost: `find where name="x"` inside `[…]` + * *does* lower to a real `(= $name x)` node, and those `=` bytes now stay + * `unclassified` — 224 of the corpus's 1,259 bracket `=` against 1,035 that + * were plain `arg=value`. Abstaining on all of them beats claiming 82% + * wrong; a `where`-aware fill can take them later. + * 2. **Glued after `=` is an argument value.** `in-interface-list=!LAN`, + * `.id=*2`, `oid=.1.3.6.1.2.1` — the byte after an argument `=` starts the + * value, never an operator. + * 3. **Glued into an argument name is a name byte.** `:foreach x in=$list` is + * `/foreach counter=$x` in the IL with **no** `(in …)` node anywhere — `in=` + * is an argument name. Same for the dotted names: the IL keeps + * `security.authentication-types=wpa2-psk` as one name and renders `.id` as + * the single symbol `$.id`, never a `(. …)` concat. + * + * Vocabulary is provisional: one `operator` class for all 26 spellings + the + * two aliases (`&&`, `||`). Per-operator/per-category legend is #264 B5. + * `<>` is **not** one token — it re-lexes to `<` then `>` as two tokens. + * `syntax-meta` is residual and merged, never a source (#255). + */ + +import type { ExplainToken } from "../explain.ts"; +import { loweredSpellings, routerosOperators } from "./operators.ts"; +import { scanQuotedString } from "./quoted-string.ts"; + +const WORD_OPERATORS = new Set(["and", "or", "in", "any"]); +/** + * Spellings that are only operators inside an expression group — everywhere + * else they are path separators, argument separators or hyphens in a name. + * `->` is never ambiguous and is deliberately absent. + */ +const EXPRESSION_ONLY = new Set([",", "/", "=", "-"]); + +/** The only opener that starts an expression. `[` is a command, `{` a block. */ +const EXPRESSION_OPENER = "("; +const OPENERS = "([{"; +const CLOSERS = ")]}"; + +const PUNCT_ONLY_DOT_GUARD = "."; + +function isDigit(c: string): boolean { + return c >= "0" && c <= "9"; +} + +function isSpace(c: string): boolean { + return c === " " || c === "\t" || c === "\r" || c === "\n"; +} + +// Spellings to scan for. Hard-coded from the grounded table; no regex. +function buildSpellings(): string[] { + const ops = routerosOperators().map((o) => o.spelling); + const aliases = loweredSpellings() + .filter((e) => e.kind === "alias") + .map((e) => e.spelling); + // `<>` is re-lexed, not one token — exclude it. + const all = [...ops, ...aliases].filter((s) => s !== "<>"); + // Deduplicate, longest first. + const uniq = [...new Set(all)]; + uniq.sort((a, b) => b.length - a.length || b.localeCompare(a)); + return uniq; +} + +const SPELLINGS = buildSpellings(); + +function isWordOperator(spelling: string): boolean { + return WORD_OPERATORS.has(spelling); +} + +/** + * `=` immediately to the left — everything after an argument `=` is the value + * (`in-interface-list=!LAN`, `.id=*2`, `oid=.1.3.6.1.2.1`), never an operator. + */ +function followsArgumentEquals(analyzed: string, start: number): boolean { + return start > 0 && analyzed[start - 1] === "="; +} + +/** + * The spelling is glued into an argument NAME, which ends at its `=`. + * + * Two grounded shapes: a word operator immediately before `=` (`in=$list` in + * `:foreach`), and a dot that joins name parts (`.id=`, `.passphrase=`, + * `configuration.ssid=`, `security.authentication-types=`). Deliberately not + * generalized to every spelling — `<`/`>`/`-` before an `=` have no grounded + * name shape and generalizing would abstain on real comparisons. + */ +function insideArgumentName( + analyzed: string, + start: number, + spelling: string, +): boolean { + const after = start + spelling.length; + if (isWordOperator(spelling)) return analyzed[after] === "="; + if (spelling !== PUNCT_ONLY_DOT_GUARD) return false; + // `.` + at least one name character, then the `=` that ends the name. + let p = after; + if (p >= analyzed.length || !/[A-Za-z]/.test(analyzed[p] as string)) + return false; + while (p < analyzed.length && /[A-Za-z0-9._-]/.test(analyzed[p] as string)) + p++; + return analyzed[p] === "="; +} + +function isAllResidual( + start: number, + len: number, + isResidual: (pos: number) => boolean, +): boolean { + for (let p = start; p < start + len; p++) if (!isResidual(p)) return false; + return true; +} + +/** + * Operator spans on the residual. + * + * `analyzed` is the ASCII-normalized document text; `residual` is the gap set + * left by earlier fills (already sorted, no overlaps). Every emitted span's + * bytes are fully inside `residual`, sorted by `start`, non-overlapping, and + * carry `class: "operator"` + `ev: "e10"`. + */ +export function operatorSpans( + analyzed: string, + residual: readonly { start: number; end: number }[], +): ExplainToken[] { + const len = analyzed.length; + if (len === 0 || residual.length === 0) return []; + + // Fast residual membership — residual is sorted. + function isResidual(pos: number): boolean { + // Binary search over residual ranges. + let lo = 0; + let hi = residual.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const r = residual[mid] as { start: number; end: number }; + if (pos < r.start) hi = mid - 1; + else if (pos >= r.end) lo = mid + 1; + else return true; + } + return false; + } + + const out: ExplainToken[] = []; + // Delimiter stack, not a depth counter: only `(` opens an expression, so the + // innermost opener — not the nesting level — decides `EXPRESSION_ONLY`. + const openers: string[] = []; + let i = 0; + + while (i < len) { + const ch = analyzed[i] as string; + + // String interior — skip entirely, no delimiter accounting inside. + if (ch === '"' && isResidual(i)) { + const scan = scanQuotedString(analyzed, i); + if (scan.end > i + 1) { + i = scan.end; + continue; + } + // Unterminated double-quoted string — advance one so the scan makes progress + i++; + continue; + } + + const residualAt = isResidual(i); + + // Delimiter tracking — only when the byte itself is residual. A `(` or `[` + // inside a comment/variable is not residual and does not open anything. + // This is the conservatism signal for `, / = -`. + if (residualAt && OPENERS.includes(ch)) { + openers.push(ch); + i++; + continue; + } + if (residualAt && CLOSERS.includes(ch)) { + openers.pop(); + i++; + continue; + } + + if (!residualAt) { + i++; + continue; + } + + // Everything after an argument `=` is that argument's value — no operator + // starts there. Checked once per offset, not once per spelling. + if (followsArgumentEquals(analyzed, i)) { + i++; + continue; + } + + // Try longest-match operator at i. + let matched: string | null = null; + for (const spell of SPELLINGS) { + if (i + spell.length > len) continue; + // Quick first-char filter + if (analyzed[i] !== spell[0]) continue; + if (analyzed.slice(i, i + spell.length) !== spell) continue; + if (!isAllResidual(i, spell.length, isResidual)) continue; + + if (isWordOperator(spell)) { + const before = i > 0 ? analyzed[i - 1] : undefined; + const after = + i + spell.length < len ? analyzed[i + spell.length] : undefined; + if (before !== undefined && /[A-Za-z0-9._-]/.test(before)) continue; + if (after !== undefined && /[A-Za-z0-9._-]/.test(after)) continue; + } + + if (spell === PUNCT_ONLY_DOT_GUARD) { + // `1.` variable name — dot glued to left alnum and followed by + // space / close delimiter / end. (1. 2) -> $1. juxtaposition. + if (i > 0 && /[A-Za-z0-9]/.test(analyzed[i - 1] as string)) { + const right = i + 1 < len ? analyzed[i + 1] : undefined; + if ( + right === undefined || + isSpace(right) || + right === ")" || + right === "]" || + right === "}" || + right === ";" || + right === "," || + right === '"' || + right === "'" + ) { + continue; + } + } + // Tight digit.digit — IP literal 1.2, no spaces. + if ( + i > 0 && + i + 1 < len && + isDigit(analyzed[i - 1] as string) && + isDigit(analyzed[i + 1] as string) + ) { + continue; + } + // Time literal (.1) — dot immediate after '(' and before digit. + if ( + i > 0 && + analyzed[i - 1] === "(" && + i + 1 < len && + isDigit(analyzed[i + 1] as string) + ) { + continue; + } + // Second byte of `..` — variable `$.` re-lex. + if (i > 0 && analyzed[i - 1] === ".") continue; + } + + if (spell === "/") { + if (i > 0 && analyzed[i - 1] === "/") continue; + } + + // Glued into an argument name (`in=`, `.id=`, `configuration.ssid=`). + if (insideArgumentName(analyzed, i, spell)) continue; + + // Expression-only conservatism — leaves `, / = -` for the path/arg + // fills everywhere except directly inside `(`. `->` is always allowed. + if ( + EXPRESSION_ONLY.has(spell) && + openers[openers.length - 1] !== EXPRESSION_OPENER + ) + continue; + + matched = spell; + break; + } + + if (matched !== null) { + out.push({ + start: i, + end: i + matched.length, + class: "operator" as const, + ev: "e10", + }); + i += matched.length; + continue; + } + + i++; + } + + return out; +} diff --git a/test/fixtures/explain/tokens.json b/test/fixtures/explain/tokens.json index e41a533..301f430 100644 --- a/test/fixtures/explain/tokens.json +++ b/test/fixtures/explain/tokens.json @@ -2,27 +2,29 @@ "corpus": { "sourceScripts": 948, "totalBytes": 1426731, - "classifiedBytes": 391474, - "unclassifiedBytes": 1035257, - "classifiedPct": 27.438529056984112, - "totalTokens": 46580, - "avgTokensPerScript": 49.13502109704641, + "classifiedBytes": 403516, + "unclassifiedBytes": 1023215, + "classifiedPct": 28.28255641743258, + "totalTokens": 63416, + "avgTokensPerScript": 66.89451476793249, "classCounts": { "comment": 5545, - "unclassified": 23291, + "unclassified": 30709, "variable-global": 2769, "variable-local": 11331, "variable-parameter": 2242, - "variable-auto": 1402 + "variable-auto": 1402, + "operator": 9418 }, "classByteCounts": { "comment": 277806, - "unclassified": 1035257, + "unclassified": 1023215, "variable-global": 27428, "variable-local": 71695, "variable-parameter": 10277, - "variable-auto": 4268 + "variable-auto": 4268, + "operator": 12042 } }, - "_note": "token partition census \u2014 see scripts/explain-token-census.ts; re-derive with bun run explain:token-census --json (wrapped as {corpus: ...})" + "_note": "token partition census — see scripts/explain-token-census.ts; re-derive with bun run explain:token-census --json (wrapped as {corpus: ...})" } diff --git a/test/unit/explain-token-census.test.ts b/test/unit/explain-token-census.test.ts index 4614fc8..7ff7ab2 100644 --- a/test/unit/explain-token-census.test.ts +++ b/test/unit/explain-token-census.test.ts @@ -22,6 +22,7 @@ import { explainCommand, explainEnvelope, renderExplainEnvelope, + residualRanges, } from "../../src/explain.ts"; const README = readFileSync( @@ -283,3 +284,99 @@ describe("#289 B1 — token-census drift gate", () => { ).toEqual([]); }); }); + +describe("#290 B2 — residualRanges seam and multi-fill buildTokens", () => { + test("residual of empty claimed is the whole input", () => { + expect(residualRanges(5, [])).toEqual([{ start: 0, end: 5 }]); + expect(residualRanges(0, [])).toEqual([]); + }); + + test("residual coalesces and sorts claimed spans", () => { + expect( + residualRanges(6, [ + { start: 4, end: 6 }, + { start: 0, end: 2 }, + ]), + ).toEqual([{ start: 2, end: 4 }]); + }); + + test("buildTokens output is byte-ordered whatever order the fills arrive in", () => { + // Fill order IS the fill order (#290 design decision 1), but it decides + // who may CLAIM a byte — not the emitted order. The partition is always + // sorted by `start`, so passing the same two fills either way round is the + // same stream. A fill offering only residual can never overlap; the throw + // below is the safety net for a caller that violates that. + const op: ExplainSpan = { start: 2, end: 4, class: "comment", ev: "e10" }; + const comment: ExplainSpan = { + start: 0, + end: 2, + class: "comment", + ev: "e2", + }; + const spansFirst = buildTokens("abcdef", [[comment], [op]]); + const opFirst = buildTokens("abcdef", [[op], [comment]]); + expect(spansFirst).toEqual(opFirst); + expect(spansFirst.map((t) => t.start)).toEqual([0, 2, 4]); + // The trailing gap is `unclassified`, attributed to the coordinates pass. + expect(spansFirst.map((t) => t.ev)).toEqual(["e2", "e10", "e1"]); + expect(spansFirst[2]?.class).toBe("unclassified"); + // Overlap across fills is a hard throw, not a silent merge. + expect(() => + buildTokens("abcdef", [ + [{ start: 0, end: 4, class: "comment", ev: "e2" }], + [{ start: 2, end: 6, class: "comment", ev: "e10" }], + ]), + ).toThrow(/overlapping spans/); + }); + + test("operator class is a first-class token class with ev e10", () => { + const data = explainCommand(":put (1+2)", { tokens: true }); + const ops = (data.tokens ?? []).filter((t) => t.class === "operator"); + expect(ops.length).toBeGreaterThan(0); + for (const t of ops) expect(t.ev).toBe("e10"); + // `spans` stays proof-only — no operator class there. + expect(data.spans.some((s) => (s.class as string) === "operator")).toBe( + false, + ); + // Evidence cites e10 only when operator tokens exist. + const hasOperatorEvidence = data.evidence.some((e) => e.id === "e10"); + expect(hasOperatorEvidence).toBe(true); + const noOp = explainCommand("/ip address add address=1.1.1.1", { + tokens: true, + }); + expect(noOp.evidence.some((e) => e.id === "e10")).toBe(false); + expect(fixture.corpus.classCounts["operator"]).toBeGreaterThan(0); + expect(fixture.corpus.classByteCounts["operator"]).toBeGreaterThan(0); + }); + + test("fill order conservatism: top-level , / = - stay unclassified for path/arg fills", () => { + const commaTop = explainCommand(":put 1,2", { tokens: true }); + expect( + commaTop.tokens?.some((t) => t.class === "operator" && t.start === 7), + ).toBe(false); + const commaInside = explainCommand(":put (1,2)", { tokens: true }); + expect( + commaInside.tokens?.some((t) => t.class === "operator" && t.start === 7), + ).toBe(true); + // Slash and equals share the same conservatism. + expect( + explainCommand(":put 1 / 2", { tokens: true }).tokens?.some( + (t) => t.class === "operator", + ), + ).toBe(false); + expect( + explainCommand(":put (1 / 2)", { tokens: true }).tokens?.some( + (t) => t.class === "operator", + ), + ).toBe(true); + // …and a `[ ]` is a command substitution, so it does NOT lift the + // conservatism the way a `( )` does: every byte of this line is path or + // argument structure and none of it is an operator. + expect( + explainCommand( + ":put [/ip/route/find where dst-address=0.0.0.0/0 gateway-status=reachable]", + { tokens: true }, + ).tokens?.some((t) => t.class === "operator"), + ).toBe(false); + }); +}); diff --git a/test/unit/operator-tokens.test.ts b/test/unit/operator-tokens.test.ts new file mode 100644 index 0000000..828c20f --- /dev/null +++ b/test/unit/operator-tokens.test.ts @@ -0,0 +1,245 @@ +/** + * #290 B2 — operatorSpans traps. + * + * Every row in the table is a device-measured trap from #255 + #290. + * Vocabulary is provisional: one `operator` class (ev e10), `<>` re-lexes to + * two tokens, `&&`/`||` lower but get spans, and the scanner never claims a + * byte that `spans[]` already holds. + * + * The fill runs on the residual only — fill order is structural, not lexical + * (#290 design decision 1). Its three abstentions — `, / = -` outside `( )`, + * bytes glued after an argument `=`, and bytes glued into an argument name — + * are asserted here and via explainCommand. Each is grounded on the corpus + * device oracle; see the `operator-tokens.ts` header for the IL evidence. + */ + +import { describe, expect, test } from "bun:test"; +import { analyzeCoordinates } from "../../src/explain/coordinates.ts"; +import { operatorSpans } from "../../src/explain/operator-tokens.ts"; +import { explainCommand, residualRanges } from "../../src/explain.ts"; + +function opsViaExplain(input: string): string[] { + const data = explainCommand(input, { tokens: true }); + const analyzed = new TextDecoder().decode(analyzeCoordinates(input).analyzed); + return (data.tokens ?? []) + .filter((t) => t.class === "operator") + .map((t) => analyzed.slice(t.start, t.end)); +} + +function opsDirect( + analyzed: string, + residual: { start: number; end: number }[], +): string[] { + return operatorSpans(analyzed, residual).map((s) => + analyzed.slice(s.start, s.end), + ); +} + +describe("#290 operator fill — direct residual scanner", () => { + test("longest-match: <= before <, << before <, -> before -", () => { + expect(opsDirect("a<=b", [{ start: 0, end: 4 }])).toEqual(["<="]); + expect(opsDirect("a<b", [{ start: 0, end: 4 }])).toEqual(["->"]); + expect(opsDirect("a<%%b", [{ start: 0, end: 5 }])).toEqual(["<%%"]); + }); + + test("word-bounded: and/or/in/any only when not part of a word", () => { + // Residual is whole input for these direct tests. + expect(opsDirect("and", [{ start: 0, end: 3 }])).toEqual(["and"]); + expect(opsDirect("xand", [{ start: 0, end: 4 }])).toEqual([]); + expect(opsDirect("andx", [{ start: 0, end: 4 }])).toEqual([]); + expect(opsDirect("a and b", [{ start: 0, end: 7 }])).toEqual(["and"]); + // reads-as-variable spellings never emitted even though word-shaped + expect(opsDirect("not", [{ start: 0, end: 3 }])).toEqual([]); + expect(opsDirect("xor", [{ start: 0, end: 3 }])).toEqual([]); + }); + + test("respects residual — never claims a byte outside it", () => { + // Simulate a prior fill claiming [1,3) + expect( + opsDirect("a+b", [ + { start: 0, end: 1 }, + { start: 3, end: 3 }, + ]), + ).toEqual([]); + expect(opsDirect("a+b", [{ start: 0, end: 3 }])).toEqual(["+"]); + }); + + test("skips quoted strings, no depth accounting inside", () => { + expect(opsDirect('"a + b" + c', [{ start: 0, end: 10 }])).toEqual(["+"]); + }); + + test("dot traps: 1. variable, 1.2 IP, (.1) time, .. second dot", () => { + // 1. variable — dot glued to left alnum and followed by space + expect(opsDirect("(1. 2)", [{ start: 0, end: 6 }])).toEqual([]); + // IP literal tight digit.digit + expect(opsDirect("(1.2)", [{ start: 0, end: 5 }])).toEqual([]); + // Time literal (.1) + expect(opsDirect("(.1)", [{ start: 0, end: 4 }])).toEqual([]); + // .. -> only first dot + expect(opsDirect("a..b", [{ start: 0, end: 4 }])).toEqual(["."]); + // Leading dot with space separation is operator + expect(opsDirect("(1 .2)", [{ start: 0, end: 6 }])).toEqual(["."]); + expect(opsDirect("(1 . 2)", [{ start: 0, end: 7 }])).toEqual(["."]); + }); + + test("slash trap: second byte of //", () => { + expect(opsDirect("(a//b)", [{ start: 0, end: 6 }])).toEqual(["/"]); + // Top-level slash is left for path fill — not claimed here. + expect(opsDirect("a//b", [{ start: 0, end: 4 }])).toEqual([]); + }); + + test("expression conservatism: , / = - only directly inside ( )", () => { + // residual is whole input; no opener so these are skipped + expect(opsDirect("a,b", [{ start: 0, end: 3 }])).toEqual([]); + expect(opsDirect("a/b", [{ start: 0, end: 3 }])).toEqual([]); + expect(opsDirect("a=b", [{ start: 0, end: 3 }])).toEqual([]); + expect(opsDirect("a-b", [{ start: 0, end: 3 }])).toEqual([]); + // Inside parens + expect(opsDirect("(a,b)", [{ start: 0, end: 5 }])).toEqual([","]); + expect(opsDirect("(a/b)", [{ start: 0, end: 5 }])).toEqual(["/"]); + expect(opsDirect("(a=b)", [{ start: 0, end: 5 }])).toEqual(["="]); + expect(opsDirect("(a-b)", [{ start: 0, end: 5 }])).toEqual(["-"]); + // -> is allowed with no opener at all + expect(opsDirect("a->b", [{ start: 0, end: 4 }])).toEqual(["->"]); + }); + + test("`[` is command substitution, not an expression group", () => { + // The IL for a bracketed command is `(evl /path/segments arg=value)` — + // the `/` are path separators and the `=` an argument separator, with no + // division and no comparison node. So `, / = -` abstain inside `[ ]`. + expect(opsDirect("[a,b]", [{ start: 0, end: 5 }])).toEqual([]); + expect( + opsDirect("[/ip/route/find where x=1]", [{ start: 0, end: 25 }]), + ).toEqual([]); + expect( + opsDirect("[/interface/get $x default-name]", [{ start: 0, end: 32 }]), + ).toEqual([]); + // A `(` nested inside `[ ]` restores expression context. + expect(opsDirect("[f ($a/$b)]", [{ start: 0, end: 11 }])).toEqual(["/"]); + // …and a `[ ]` nested inside `(` takes it away again. + expect( + opsDirect("(x . [/ip/route/print])", [{ start: 0, end: 23 }]), + ).toEqual(["."]); + // `{` (block or array literal) is command context too. + expect( + opsDirect("do={/ip/route/add x=1}", [{ start: 0, end: 22 }]), + ).toEqual([]); + // Non-ambiguous spellings are unaffected by the opener. + expect(opsDirect("[$a->1]", [{ start: 0, end: 7 }])).toEqual(["->"]); + expect(opsDirect("[find where a>1]", [{ start: 0, end: 16 }])).toEqual([ + ">", + ]); + }); + + test("glued after `=` is an argument value, never an operator", () => { + // `!LAN`, `*2`, `.1.3.6.1…` are value bytes on the device. + expect( + opsDirect("in-interface-list=!LAN", [{ start: 0, end: 22 }]), + ).toEqual([]); + expect(opsDirect("(x=!y)", [{ start: 0, end: 6 }])).toEqual(["="]); + expect(opsDirect("[find where .id=*2]", [{ start: 0, end: 19 }])).toEqual( + [], + ); + expect(opsDirect("oid=.1.3.6.1", [{ start: 0, end: 12 }])).toEqual([]); + // A space breaks the glue — that is an ordinary operand position. + expect(opsDirect("(a = -1)", [{ start: 0, end: 8 }])).toEqual(["=", "-"]); + }); + + test("glued into an argument name is a name byte", () => { + // `:foreach x in=$list` is `/foreach counter=$x` in the IL — no `(in …)`. + expect(opsDirect("in=$list", [{ start: 0, end: 8 }])).toEqual([]); + expect(opsDirect("(a in $b)", [{ start: 0, end: 9 }])).toEqual(["in"]); + // Dotted argument names stay one name: `.id=`, `configuration.ssid=`. + expect(opsDirect("[find where .id=1]", [{ start: 0, end: 18 }])).toEqual( + [], + ); + expect( + opsDirect("[find where configuration.ssid=$s]", [{ start: 0, end: 34 }]), + ).toEqual([]); + expect( + opsDirect("[set x security.authentication-types=wpa2]", [ + { start: 0, end: 42 }, + ]), + ).toEqual([]); + // The dot abstains on its own merit — inside `( )` the `=` is still a + // comparison, but the name-joining dot is not concatenation. + expect( + opsDirect("(configuration.ssid=$s)", [{ start: 0, end: 23 }]), + ).toEqual(["="]); + // A dot NOT ending at an `=` is still concatenation. + expect(opsDirect("(a . b)", [{ start: 0, end: 7 }])).toEqual(["."]); + expect(opsDirect("(a .b)", [{ start: 0, end: 6 }])).toEqual(["."]); + }); + + test("lowered spellings: && and || as length-2 operators, <> as two tokens", () => { + expect(opsDirect("a && b", [{ start: 0, end: 6 }])).toEqual(["&&"]); + expect(opsDirect("a || b", [{ start: 0, end: 6 }])).toEqual(["||"]); + expect(opsDirect("1<>2", [{ start: 0, end: 4 }])).toEqual(["<", ">"]); + }); +}); + +describe("#290 operator fill — via explainCommand (masking + depth)", () => { + test.each([ + [":put (1+2)", ["+"]], + [":put (1 .2)", ["."]], + [":put (1. 2)", []], + [":put (1.2)", []], + [":put (.1)", []], + [":put (a..b)", ["."]], + [":put (a//b)", ["/"]], + [":put (a && b)", ["&&"]], + [":put (a || b)", ["||"]], + [":put (1<>2)", ["<", ">"]], + [":put (a and b)", ["and"]], + [':put "a + b" + 1', ["+"]], + ["# comment + plus\n:put (1+2)", ["+"]], + [":put (1,2)", [","]], + [":put 1,2", []], + [":put (1<=2)", ["<="]], + [":put (1<<2)", ["<<"]], + [":put (1->2)", ["->"]], + [":put (1<%%2)", ["<%%"]], + [":put (1 / 2)", ["/"]], + [":put 1 / 2", []], + ])("%s → %j", (input, expected) => { + expect(opsViaExplain(input as string)).toEqual(expected as string[]); + }); + + test("variable spans mask operators — $x + 1 claims $x bytes first", () => { + // $x is a variable-local span — its bytes are not residual. The `$` byte + // itself is operator-agnostic; the `x` byte is claimed by the symbol pass. + const input = ":local x 1; :put ($x + 1)"; + const data = explainCommand(input, { tokens: true }); + const analyzed = new TextDecoder().decode( + analyzeCoordinates(input).analyzed, + ); + const tokens = data.tokens ?? []; + const plus = tokens.filter((t) => analyzed.slice(t.start, t.end) === "+"); + expect(plus.length).toBe(1); + expect(plus[0]?.class).toBe("operator"); + // The variable `x` at 19 is not an operator token. + const varAt19 = tokens.find((t) => t.start === 19 && t.end === 20); + expect(varAt19?.class).toBe("variable-local"); + }); + + test("every operator token carries ev e10 and evidence cites it", () => { + const data = explainCommand(":put (1+2*3)", { tokens: true }); + const ops = (data.tokens ?? []).filter((t) => t.class === "operator"); + expect(ops.length).toBe(2); + for (const t of ops) expect(t.ev).toBe("e10"); + expect(data.evidence.some((e) => e.id === "e10")).toBe(true); + // No operator → no e10 + const noOp = explainCommand("/ip address add address=1.1.1.1", { + tokens: true, + }); + expect(noOp.evidence.some((e) => e.id === "e10")).toBe(false); + }); + + test("residualRanges is the complement — used by fill order", () => { + expect(residualRanges(5, [{ start: 1, end: 3 }])).toEqual([ + { start: 0, end: 1 }, + { start: 3, end: 5 }, + ]); + }); +});