diff --git a/GLOSSARY.txt b/GLOSSARY.txt index f7b573b..320fc58 100644 --- a/GLOSSARY.txt +++ b/GLOSSARY.txt @@ -709,3 +709,6 @@ andx alnum wpa # `security.authentication-types=wpa2-psk` — a wifi argument name whose interior dot the operator fill must not read as concatenation psk # pre-shared key — the value half of the same grounded `wpa2-psk` example + +# #293 B2 arg-fill vocabulary +retag # B5 may retag the `=` separator from the provisional `arg` class without moving byte coverage diff --git a/commands/explain/README.md b/commands/explain/README.md index 37ff656..e7a24df 100644 --- a/commands/explain/README.md +++ b/commands/explain/README.md @@ -1206,8 +1206,8 @@ And the grounded complement — asked, and refused: `bun run explain:token-census:readme` and gated against it by `bun run explain:token-census:readme:check`; the fixture itself is gated against a fresh corpus run by `bun run explain:token-census:check`. Of - 1,426,731 analyzed bytes, 403,516 are classified (28.28%), the remaining - 1,023,215 are `unclassified`. The census emits 63,416 tokens (avg 66.9 per + 1,426,731 analyzed bytes, 497,801 are classified (34.89%), the remaining + 928,930 are `unclassified`. The census emits 86,290 tokens (avg 91.0 per script). Every byte belongs to exactly one token — sorted by `start`, no gaps, no overlaps, `join(slice) === input` — and the `class` field is provisional until #264 B5. Each B2 fill should move the classified @@ -1219,14 +1219,25 @@ partition whose `class` is provisional until #264 B5 (every unclaimed byte is `unclassified`). Since B1 its only fill source was `data.spans[]` (comment runs and resolved variable occurrences); **`#290`'s operator fill is the first B2 fill** — `src/explain/operator-tokens.ts` claims `operator` bytes on the -residual left by spans, with fill order enforced by argument order to -`buildTokens` ([#290 design decision 1](../../src/explain.ts)). -`ExplainTokenClass` is `ExplainSpanClass | "operator" | "unclassified"` — one -provisional `operator` class for all 26 spellings + 2 aliases, not per-operator -or per-category (`#264` B5). `ExplainSpanClass` and `data.spans[]` stay -proof-only; `src/explain/operators.ts` remains data plus accessors, and the -operator table above is its source. The `centrs → highlight` projection is B4 -and reads both. +residual left by spans — and **`#293`'s arg fill is the second** — +`src/explain/arg-tokens.ts` claims argument names and their `=` (name run +`[span.start, valueSpan.start - 1)` plus the single `=` byte at +`valueSpan.start - 1`; the value itself, `valueSpan`, is left for the next +fill) on the residual left by spans, before operators see it. Fill order is +enforced by argument order to `buildTokens` +([#290 design decision 1](../../src/explain.ts)): `spans` claims first, then +`arg`, then `operator` sees only what neither wanted — which is why the +operator fill can abstain on `, / = -` outside `( )`, on bytes glued after an +argument `=`, and on bytes glued into an argument name without a smarter +operator scanner. +`ExplainTokenClass` is `ExplainSpanClass | "operator" | "arg" | "unclassified"` +— one provisional `operator` class for all 26 spellings + 2 aliases plus one +provisional `arg` class for both the name bytes and the `=` separator (emit +first, name later — whether the `=` later deserves its own class is #264 B5 and +does not move the byte coverage), not per-operator or per-category (`#264` B5). +`ExplainSpanClass` and `data.spans[]` stay proof-only; `src/explain/operators.ts` +remains data plus accessors, and the operator table above is its source. The +`centrs → highlight` projection is B4 and reads both. **Where the operator fill abstains.** Fill order is the resolution mechanism, so a byte that is structurally part of a path or an argument is left diff --git a/src/explain.ts b/src/explain.ts index 1d1d8f4..bbcb80b 100644 --- a/src/explain.ts +++ b/src/explain.ts @@ -89,6 +89,7 @@ import { canonicalizeExecuteCommand, isWriteShaped, } from "./execute.ts"; +import { argSpans } from "./explain/arg-tokens.ts"; import { type ArgumentKind, lexArguments, @@ -416,8 +417,8 @@ export interface ExplainSpan extends ExplainSpanRange { * (#290 design decision 1) and a structural ambiguity (`/` path sep vs `division`) * resolves by which analyzer came first rather than by a smarter byte scanner. * - * A fill is typed as `ExplainToken[]` because B2 fills (operator, then - * path/arg) introduce classes outside `ExplainSpanClass`; the proof-only + * A fill is typed as `ExplainToken[]` because B2 fills (operator, arg, then + * path) introduce classes outside `ExplainSpanClass`; the proof-only * `spans[]` (ExplainSpan[]) remains a subtype and so fits the same slot. */ export type TokenFill = readonly ExplainToken[]; @@ -435,8 +436,21 @@ export type TokenFill = readonly ExplainToken[]; * not the final LSP/SCIP legend, and every byte no analyzer claims becomes * `unclassified`. Filling those holes is B2, one PR per fill. `unclassified` * is a first-class answer, not a placeholder to be avoided. + * + * B2 so far: `operator` (26 spellings + 2 aliases, `syntax-meta` is a residual + * merge, never a source, #255) and `arg` (argument names and their `=` as + * located by `args.ts` — the name run is `[span.start, valueSpan.start - 1)` + * and the `=` is the single byte at `valueSpan.start - 1`; the value itself is + * `valueSpan` and is left for the next fill, #293). `arg` covers both the name + * bytes and the `=` byte in one provisional class (emit first, name later); + * whether the `=` later deserves its own class is #264 B5 vocabulary and does + * not move the byte coverage. */ -export type ExplainTokenClass = ExplainSpanClass | "operator" | "unclassified"; +export type ExplainTokenClass = + | ExplainSpanClass + | "operator" + | "arg" + | "unclassified"; export interface ExplainToken extends ExplainSpanRange { /** @@ -617,6 +631,7 @@ const EV = { transport: "e8", values: "e9", operators: "e10", + args: "e11", } as const; type EvidenceKey = keyof typeof EV; @@ -699,6 +714,24 @@ const EVIDENCE: Record = { basis: "heuristic", outcome: "ok", }, + /** + * `heuristic`, not `direct`, even though the span arithmetic is exact. + * + * `basis` describes the FACT, not the arithmetic. The fact here is "these + * bytes are an argument name and its `=`", and it rests on `argsAt` — the + * verb boundary, which `EV.statements` already records as `heuristic` and + * which this type's own doc names as the example of a rule a live probe + * could overturn. A misread boundary shifts every token this fill emits. + * `operatorSpans` sits at `heuristic` on identical grounds: deterministic + * byte arithmetic over a grounded table, downstream of an offline rule. + */ + args: { + id: EV.args, + source: "canonicalizer", + probe: "argSpans", + basis: "heuristic", + outcome: "ok", + }, }; /** Diagnostic rendering for each defect class. */ @@ -1125,29 +1158,39 @@ export function explainCommand( // `spans` (proof-only: comment + variables) claims first; every later fill // sees only the residual left by the fills before it, so a structural // ambiguity (`/` path vs division, `,` arg sep vs concat) resolves by which - // analyzer came first. `operatorSpans` is the first such fill. - // - // **The path and arg fills belong BEFORE `operatorSpans`, not after.** The - // intended end state is that `pathresolve.ts` has already claimed the `/` - // and `args.ts` the `=` by the time the operator scanner runs, so it only - // ever sees bytes nobody else wanted. Until those fills exist, the operator - // fill buys the same safety by abstaining wherever a byte is structurally - // path or argument (see `operator-tokens.ts` — three grounded abstentions). - // Inserting a fill after it would leave those abstentions doing work they - // should not have to do, and the scanner can relax them only once the fill - // that owns those bytes runs first: + // analyzer came first. `argSpans` is the second such fill, `operatorSpans` + // the third — the intended end state is + // `const fills: TokenFill[] = [spans, argSpans, opSpans];` so `args.ts` has + // claimed the `=` before the operator scanner runs (#293 design decision 1). + // Until the path fill exists, the operator fill still abstains on `, / -` + // outside `( )` wherever a path byte would be, but for `=` the arg fill now + // owns the byte first, so the operator conservatism is no longer load-bearing + // for `=` (it stays for `, / -`). Inserting a fill after `operatorSpans` + // would leave those abstentions doing work they should not have to do, and + // the scanner can relax them only once the fill that owns those bytes runs + // first: // const pathSpans = pathSpansOnResidual(analyzed, residual0, ...); // const residual1 = residualRanges(analyzed.length, [...spans, ...pathSpans]); - // const opSpans = operatorSpans(analyzed, residual1); - // const fills: TokenFill[] = [spans, pathSpans, opSpans]; + // const argSpansList = argSpans(analyzed, residual1, argCandidates); + // const residual2 = residualRanges(analyzed.length, [...spans, ...pathSpans, ...argSpansList]); + // const opSpans = operatorSpans(analyzed, residual2); + // const fills: TokenFill[] = [spans, pathSpans, argSpansList, opSpans]; // // Gate the scan on `options.tokens` — no residual work when the caller // did not ask for `data.tokens`. Future B2 fills belong inside this branch. let tokens: ExplainToken[] | undefined; if (options.tokens === true) { const residual0 = residualRanges(analyzed.length, spans); - const opSpans = operatorSpans(analyzed, residual0); - const fills: TokenFill[] = [spans, opSpans]; + const argCandidates = statements.flatMap((s) => + s.arguments?.read === true ? s.arguments.tokens : [], + ); + const argSpansList = argSpans(analyzed, residual0, argCandidates); + const residual1 = residualRanges(analyzed.length, [ + ...spans, + ...argSpansList, + ]); + const opSpans = operatorSpans(analyzed, residual1); + const fills: TokenFill[] = [spans, argSpansList, opSpans]; tokens = buildTokens(analyzed, fills); } diff --git a/src/explain/arg-tokens.ts b/src/explain/arg-tokens.ts new file mode 100644 index 0000000..56f2a5c --- /dev/null +++ b/src/explain/arg-tokens.ts @@ -0,0 +1,144 @@ +/** + * B2 arg fill — claims argument names and their `=` on the residual. + * + * This is the second B2 fill of #264, added after the operator fill. It runs + * **after** the proof-only spans (`comment` + `variable-*`) and **before** the + * operator fill, so the fill order is structural: `spans` claims first, then + * `arg`, then `operator` sees only what neither wanted. That ordering is why the + * operator fill can abstain on `, / = -` outside `( )`, on bytes glued after + * an argument `=`, and on bytes glued into an argument name — those bytes + * belong to a later fill, not to a smarter operator scanner. + * + * The source is `src/explain/args.ts`'s `Argument` — `span` and `valueSpan` + * already rebased into document analyzed-byte space by `src/explain.ts`. The + * name run is `[span.start, valueSpan.start - 1)` and the `=` is the single + * byte at `valueSpan.start - 1`; the value itself is `valueSpan` and is left + * for the next fill (see #293). A `positional` has no `name`/`valueSpan` and a + * `query` is a `?` word whose interior `args.ts` explicitly refuses to split + * (see `Argument.value`'s doc comment), so both are ignored — the `=` is + * derived from the token shape, never by scanning for the byte. + * + * `Argument.value` is not `Argument.text` and is never read: `value` is absent + * whenever the source spells a substitution or an escape this phase does not + * decode. This slice claims `span`/`valueSpan` *positions* only. + * + * A statement whose bytes were normalized is not addressable: `explain.ts` + * already refuses those (`its text was normalized`), and this fill inherits the + * refusal — it only sees tokens from `read === true` statements, which are + * exactly the addressable ones. + * + * Vocabulary is provisional until #264 B5: one `arg` class for both the name + * bytes and the `=` separator (design decision 2 of #293 — emit first, name + * later; `highlight`'s `syntax-meta` is a residual merge and never a source). + * If B5 splits the `=` into its own class, the byte coverage does not move, + * only the retag. + */ + +import type { ExplainArgumentToken, ExplainToken } from "../explain.ts"; + +export type ArgCandidate = ExplainArgumentToken; + +function clipToResidual( + start: number, + end: number, + residual: readonly { start: number; end: number }[], +): { start: number; end: number }[] { + if (start >= end) return []; + if (residual.length === 0) return []; + // Binary-search to first residual that could overlap `start` (r.end > start). + let lo = 0; + let hi = residual.length - 1; + let first = residual.length; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const r = residual[mid] as { start: number; end: number }; + if (r.end <= start) lo = mid + 1; + else { + first = mid; + hi = mid - 1; + } + } + const out: { start: number; end: number }[] = []; + for (let i = first; i < residual.length; i++) { + const r = residual[i] as { start: number; end: number }; + if (r.start >= end) break; + const oStart = Math.max(start, r.start); + const oEnd = Math.min(end, r.end); + if (oStart < oEnd) out.push({ start: oStart, end: oEnd }); + } + return out; +} + +/** + * Argument name + `=` spans on the residual. + * + * `analyzed` is the ASCII-normalized document text (only used for length + * checks); `residual` is the gap set left by earlier fills (sorted, no + * overlaps); `candidates` are the already-rebased `Argument` tokens from + * `read === true` statements. Every emitted span's bytes are fully inside + * `residual`, sorted by `start`, non-overlapping, and carry + * `class: "arg"` + `ev: "e11"`. + */ +export function argSpans( + analyzed: string, + residual: readonly { start: number; end: number }[], + candidates: readonly ArgCandidate[], +): ExplainToken[] { + const len = analyzed.length; + if (len === 0 || residual.length === 0 || candidates.length === 0) return []; + + const out: ExplainToken[] = []; + + for (const candidate of candidates) { + if (candidate.kind !== "attribute") continue; + if (candidate.valueSpan === undefined) continue; + if (candidate.name === undefined) continue; + const nameStart = candidate.span.start; + const eqEnd = candidate.valueSpan.start; + const eqStart = eqEnd - 1; + const nameEnd = eqStart; + // Defensive: token shape must give non-empty name and single-byte `=`. + if ( + !Number.isInteger(nameStart) || + !Number.isInteger(nameEnd) || + !Number.isInteger(eqStart) || + !Number.isInteger(eqEnd) + ) + continue; + if (nameStart < 0 || eqEnd > len) continue; + // Non-empty name. This is also what rejects `valueSpan.start === 0`, where + // `eqStart` would be -1: `nameEnd` is -1 too, so any `nameStart >= 0` trips + // it. `analyzed[-1]` on the next line would be `undefined` and continue + // anyway, so the order is safe either way — but the name check is the one + // that carries the intent. + if (nameStart >= nameEnd) continue; + // The whole token's `=` byte must be `=` in the analyzed text — a cheap + // shape check that catches a caller that passed un-rebased or misaligned + // spans. Not a scan for `=`; the position is derived from `valueSpan`. + if (analyzed[eqStart] !== "=") continue; + // Also ensure the candidate lies fully inside the analyzed length. + if (candidate.span.start < 0 || candidate.span.end > len) continue; + if (candidate.valueSpan.start < 0 || candidate.valueSpan.end > len) + continue; + + // The name run and the `=` are contiguous (`nameEnd === eqStart`), so + // clip them as ONE range per candidate. That yields the maximal `[name=]` + // run for free — split only where the residual itself has a hole — and + // makes cross-candidate fusion structurally impossible rather than merely + // unreachable: coalescing adjacent runs globally would merge two distinct + // attributes into one token if `lexArguments` ever stopped guaranteeing a + // separator between them. A `variable-*` span claiming only the `=` still + // leaves the name run behind, because the residual hole does the splitting. + for (const r of clipToResidual(nameStart, eqEnd, residual)) { + out.push({ + start: r.start, + end: r.end, + class: "arg" as const, + ev: "e11", + }); + } + } + + out.sort((a, b) => a.start - b.start || a.end - b.end); + return out; +} diff --git a/test/fixtures/explain/tokens.json b/test/fixtures/explain/tokens.json index 301f430..ae8bf65 100644 --- a/test/fixtures/explain/tokens.json +++ b/test/fixtures/explain/tokens.json @@ -2,28 +2,30 @@ "corpus": { "sourceScripts": 948, "totalBytes": 1426731, - "classifiedBytes": 403516, - "unclassifiedBytes": 1023215, - "classifiedPct": 28.28255641743258, - "totalTokens": 63416, - "avgTokensPerScript": 66.89451476793249, + "classifiedBytes": 497801, + "unclassifiedBytes": 928930, + "classifiedPct": 34.891020101196375, + "totalTokens": 86290, + "avgTokensPerScript": 91.02320675105486, "classCounts": { "comment": 5545, - "unclassified": 30709, + "unclassified": 42145, "variable-global": 2769, "variable-local": 11331, "variable-parameter": 2242, + "operator": 9418, "variable-auto": 1402, - "operator": 9418 + "arg": 11438 }, "classByteCounts": { "comment": 277806, - "unclassified": 1023215, + "unclassified": 928930, "variable-global": 27428, "variable-local": 71695, "variable-parameter": 10277, + "operator": 12042, "variable-auto": 4268, - "operator": 12042 + "arg": 94285 } }, "_note": "token partition census — see scripts/explain-token-census.ts; re-derive with bun run explain:token-census --json (wrapped as {corpus: ...})" diff --git a/test/unit/arg-tokens.test.ts b/test/unit/arg-tokens.test.ts new file mode 100644 index 0000000..1900952 --- /dev/null +++ b/test/unit/arg-tokens.test.ts @@ -0,0 +1,428 @@ +/** + * #293 B2 — argSpans traps. + * + * Second B2 fill after the operator fill. Claims argument names and their `=` + * on the residual left by `spans` (comment + variable-*), before `operatorSpans` + * sees it. Vocabulary is provisional: one `arg` class (ev e11) for both name + * bytes and the `=` separator — the `=` is derived from `valueSpan.start - 1`, + * never by scanning. + * + * Traps: positional/query have no `name`/`valueSpan` and abstain; `Argument.value` + * vs `text` is not read; normalized statements are not addressable; variable-* + * overlap is clipped to residual; coalescing `[name][=]` → one maximal run. + */ + +import { describe, expect, test } from "bun:test"; +import { argSpans } from "../../src/explain/arg-tokens.ts"; +import { analyzeCoordinates } from "../../src/explain/coordinates.ts"; +import type { ExplainArgumentToken } from "../../src/explain.ts"; +import { explainCommand, residualRanges } from "../../src/explain.ts"; + +function argsViaExplain(input: string): string[] { + const data = explainCommand(input, { tokens: true }); + const analyzed = new TextDecoder().decode(analyzeCoordinates(input).analyzed); + return (data.tokens ?? []) + .filter((t) => t.class === "arg") + .map((t) => analyzed.slice(t.start, t.end)); +} + +function argsDirect( + analyzed: string, + residual: { start: number; end: number }[], + candidates: readonly ExplainArgumentToken[], +): string[] { + return argSpans(analyzed, residual, candidates as never).map((s) => + analyzed.slice(s.start, s.end), + ); +} + +function attr( + start: number, + end: number, + valueStart: number, + valueEnd: number, + name: string, +): ExplainArgumentToken { + return { + kind: "attribute", + span: { start, end }, + name, + value: "x", + valueSpan: { start: valueStart, end: valueEnd }, + text: `${name}=x`, + }; +} + +describe("#293 arg fill — direct residual scanner", () => { + test("single attribute → single coalesced arg token [name=]", () => { + // `address` is 7 bytes, so for "address=1" the name run is [0,7), the `=` + // is the single byte [7,8) — derived as `valueSpan.start - 1` — and the + // value is valueSpan [8,9). The fill claims [0,8) as one `arg` run. + const analyzed = "address=1"; + expect( + argsDirect( + analyzed, + [{ start: 0, end: analyzed.length }], + [attr(0, 9, 8, 9, "address")], + ), + ).toEqual(["address="]); + }); + + test("coalesced even though name and = clipped separately", () => { + const analyzed = "address=1 interface=ether1"; + // Two attributes: address= at [0,8) and interface= at [10,20) + expect( + argsDirect( + analyzed, + [{ start: 0, end: analyzed.length }], + [attr(0, 9, 8, 9, "address"), attr(10, 20, 20, 26, "interface")], + ), + ).toEqual(["address=", "interface="]); + }); + + test("dotted names and hyphenated names", () => { + expect( + argsDirect( + ".proplist=name,comment", + [{ start: 0, end: 22 }], + [attr(0, 22, 10, 22, ".proplist")], + ), + ).toEqual([".proplist="]); + expect( + argsDirect( + "configuration.ssid=test", + [{ start: 0, end: 22 }], + [attr(0, 22, 19, 22, "configuration.ssid")], + ), + ).toEqual(["configuration.ssid="]); + const wpa = "security.authentication-types=wpa2-psk"; + expect( + argsDirect( + wpa, + [{ start: 0, end: wpa.length }], + [ + attr( + 0, + wpa.length, + wpa.indexOf("=") + 1, + wpa.length, + "security.authentication-types", + ), + ], + ), + ).toEqual(["security.authentication-types="]); + }); + + test("positional is ignored even with valueSpan", () => { + const analyzed = "where"; + const pos: ExplainArgumentToken = { + kind: "positional", + span: { start: 0, end: 5 }, + value: "where", + valueSpan: { start: 0, end: 5 }, + text: "where", + }; + expect(argsDirect(analyzed, [{ start: 0, end: 5 }], [pos])).toEqual([]); + }); + + test("query is ignored — name includes = but kind is query", () => { + const analyzed = "?address=1.1.1.1"; + const q: ExplainArgumentToken = { + kind: "query", + span: { start: 0, end: 16 }, + name: "address=1.1.1.1", + text: "?address=1.1.1.1", + }; + expect(argsDirect(analyzed, [{ start: 0, end: 16 }], [q])).toEqual([]); + }); + + test("attribute without valueSpan or name abstains", () => { + const analyzed = "address=1"; + const noValue: ExplainArgumentToken = { + kind: "attribute", + span: { start: 0, end: 9 }, + name: "address", + text: "address=1", + }; + expect(argsDirect(analyzed, [{ start: 0, end: 9 }], [noValue])).toEqual([]); + const noName: ExplainArgumentToken = { + kind: "attribute", + span: { start: 0, end: 9 }, + value: "1", + valueSpan: { start: 8, end: 9 }, + text: "address=1", + }; + expect(argsDirect(analyzed, [{ start: 0, end: 9 }], [noName])).toEqual([]); + }); + + test("respects residual — clipped, not re-claimed", () => { + // Simulate variable span claiming [4,8) inside "address=1" + const analyzed = "address=1"; + // name "address" at [0,7), "=" at 7. Residual missing [4,8) means "ess=" gone + expect( + argsDirect( + analyzed, + [ + { start: 0, end: 4 }, + { start: 8, end: 9 }, + ], + [attr(0, 9, 8, 9, "address")], + ), + ).toEqual(["addr"]); + // Only "=" claimed, name fully masked + expect( + argsDirect( + analyzed, + [{ start: 7, end: 8 }], + [attr(0, 9, 8, 9, "address")], + ), + ).toEqual(["="]); + // Whole name+ = outside residual → nothing + expect( + argsDirect( + analyzed, + [{ start: 9, end: 9 }], + [attr(0, 9, 8, 9, "address")], + ), + ).toEqual([]); + }); + + test("residual clipping with multiple gaps", () => { + const analyzed = "a=1 b=2 c=3"; + // residual has holes for " b=2 " interior + const residual = [ + { start: 0, end: 3 }, // "a=1" + { start: 7, end: 11 }, // " c=3" — the space at 7 plus "c=3" at [8,11) + ]; + // Provide three candidates, middle one should be fully clipped + expect( + argsDirect(analyzed, residual, [ + attr(0, 3, 2, 3, "a"), + attr(4, 7, 6, 7, "b"), + attr(8, 11, 10, 11, "c"), + ]), + ).toEqual(["a=", "c="]); + }); + + test("misaligned or out-of-bounds candidates are ignored", () => { + const analyzed = "address=1"; + // = byte not "=" in analyzed (wrongly rebased) + const badEq = attr(0, 9, 8, 9, "address"); + expect(argsDirect("address-1", [{ start: 0, end: 9 }], [badEq])).toEqual( + [], + ); + // span beyond len + expect( + argsDirect( + analyzed, + [{ start: 0, end: 9 }], + [attr(0, 99, 8, 9, "address")], + ), + ).toEqual([]); + expect( + argsDirect( + analyzed, + [{ start: 0, end: 9 }], + [attr(0, 9, 8, 99, "address")], + ), + ).toEqual([]); + }); + + test("empty inputs yield empty", () => { + expect(argsDirect("", [], [])).toEqual([]); + expect(argsDirect("", [{ start: 0, end: 0 }], [])).toEqual([]); + const analyzed = "address=1"; + expect(argsDirect(analyzed, [], [attr(0, 9, 8, 9, "address")])).toEqual([]); + expect(argsDirect(analyzed, [{ start: 0, end: 9 }], [])).toEqual([]); + }); + + test("candidates need not be sorted; output is byte-ordered", () => { + const analyzed = "b=2 a=1"; + // Provide candidates out of order + expect( + argsDirect( + analyzed, + [{ start: 0, end: 7 }], + [attr(4, 7, 6, 7, "a"), attr(0, 3, 2, 3, "b")], + ), + ).toEqual(["b=", "a="]); + }); + + test("clipToResidual binary-search path — residual far from start", () => { + // residual single far interval, candidate before it + const analyzed = `${"x".repeat(1000)}address=1`; + const base = 1000; + expect( + argsDirect( + analyzed, + [{ start: base, end: base + 9 }], + [ + attr(base, base + 9, base + 8, base + 9, "address"), + attr(0, 1, 0, 1, "x"), + ], + ), + ).toEqual(["address="]); + }); +}); + +describe("#293 arg fill — via explainCommand (masking + evidence)", () => { + test.each([ + ["/ip/address/add address=1.1.1.1", ["address="]], + [ + "/ip/address/add address=1.1.1.1 interface=ether1", + ["address=", "interface="], + ], + [ + "/ip route add dst-address=1.1.1.1 gateway=1.1.1.2", + ["dst-address=", "gateway="], + ], + ["/interface/print .proplist=name,comment", [".proplist="]], + ["/ip/address/print where address=1.1.1.1", ["address="]], + [ + "/ip/firewall/filter/add chain=forward action=accept in-interface-list=!LAN", + ["chain=", "action=", "in-interface-list="], + ], + ])("%s → %j", (input, expected) => { + expect(argsViaExplain(input as string)).toEqual(expected as string[]); + }); + + test("positional and query produce no arg tokens", () => { + expect(argsViaExplain("/ip address print")).toEqual([]); + expect(argsViaExplain("/ip address print count-only")).toEqual([]); + expect(argsViaExplain("/ip/address/print ?address=1.1.1.1")).toEqual([]); + expect(argsViaExplain("/ip address print where chain=forward")).toEqual([ + "chain=", + ]); // where is positional, chain is arg + }); + + test("all-or-nothing: variable value refuses whole statement", () => { + // gateway=$gw is a variable value → read:false → no arg tokens at all + expect( + argsViaExplain("/ip route add dst-address=1.1.1.1 gateway=$gw"), + ).toEqual([]); + expect( + argsViaExplain( + "/ip route add dst-address=1.1.1.1 gateway=$gw comment=$c", + ), + ).toEqual([]); + // substitution + expect( + argsViaExplain("/ip address add address=[/ip/route/get $x]"), + ).toEqual([]); + // array/block value + expect( + argsViaExplain("/system script add name=s source={ :put 1 }"), + ).toEqual([]); + // But without a refusing value, args appear + expect( + argsViaExplain("/ip route add dst-address=1.1.1.1 gateway=1.1.1.2"), + ).toEqual(["dst-address=", "gateway="]); + }); + + test("normalized input yields no arg tokens", () => { + expect(argsViaExplain('/system identity set name="router-🚀"')).toEqual([]); + }); + + test("value bytes are not arg — only name+=", () => { + const input = "/ip/address/add address=1.1.1.1"; + const data = explainCommand(input, { tokens: true }); + const analyzed = new TextDecoder().decode( + analyzeCoordinates(input).analyzed, + ); + const tokens = data.tokens ?? []; + // "address=" is arg, "1.1.1.1" is unclassified (value fill not yet) + const argText = tokens + .filter((t) => t.class === "arg") + .map((t) => analyzed.slice(t.start, t.end)) + .join(""); + expect(argText).toBe("address="); + // Ensure value span is not arg + const valueSlice = analyzed.slice(24, 31); + expect(valueSlice).toBe("1.1.1.1"); + const valueTokens = tokens.filter((t) => t.start >= 24 && t.end <= 31); + expect(valueTokens.every((t) => t.class !== "arg")).toBe(true); + }); + + test("every arg token carries ev e11 and evidence cites it", () => { + const data = explainCommand( + "/ip/address/add address=1.1.1.1 interface=ether1", + { tokens: true }, + ); + const args = (data.tokens ?? []).filter((t) => t.class === "arg"); + expect(args.length).toBe(2); + for (const t of args) expect(t.ev).toBe("e11"); + expect(data.evidence.some((e) => e.id === "e11")).toBe(true); + // No arg → no e11 + const noArg = explainCommand("/ip address print", { tokens: true }); + expect(noArg.evidence.some((e) => e.id === "e11")).toBe(false); + const noArg2 = explainCommand( + "/ip route add dst-address=1.1.1.1 gateway=$gw", + { tokens: true }, + ); + expect(noArg2.evidence.some((e) => e.id === "e11")).toBe(false); + }); + + test("spans stay proof-only — no arg class there", () => { + const data = explainCommand("/ip/address/add address=1.1.1.1", { + tokens: true, + }); + expect(data.spans.some((s) => (s.class as string) === "arg")).toBe(false); + }); + + test("fill order: arg owns = before operator (expression = stays operator)", () => { + // Attribute `=` is arg + expect(argsViaExplain("/ip/address/add address=1.1.1.1")).toEqual([ + "address=", + ]); + const attrEq = explainCommand("/ip/address/add address=1.1.1.1", { + tokens: true, + }); + expect( + attrEq.tokens?.some((t) => t.class === "operator" && t.start === 24), + ).toBe(false); + // Expression `=` inside ( ) is operator, not arg + const expr = explainCommand(":put (1=2)", { tokens: true }); + expect(expr.tokens?.some((t) => t.class === "operator")).toBe(true); + expect(expr.tokens?.some((t) => t.class === "arg")).toBe(false); + // Top-level slash stays unclassified for path fill, not operator + expect( + explainCommand(":put 1 / 2", { tokens: true }).tokens?.some( + (t) => t.class === "operator", + ), + ).toBe(false); + expect( + explainCommand(":put (1 / 2)", { tokens: true }).tokens?.some( + (t) => t.class === "operator", + ), + ).toBe(true); + }); + + test("residualRanges is the complement — used by fill order", () => { + expect(residualRanges(5, [{ start: 1, end: 3 }])).toEqual([ + { start: 0, end: 1 }, + { start: 3, end: 5 }, + ]); + }); + + test("mixed document: a read statement claims, an unread one does not", () => { + // The `read === true` filter is applied per STATEMENT, and the residual is + // document-global — so one refusing statement must not suppress a readable + // one, and must not let the fill spill into its own bytes. `gateway=$gw` + // is `a variable value`, which `lexArguments` refuses wholesale. + const input = "/ip/address/add address=1.1.1.1\n/ip/route/add gateway=$gw"; + const data = explainCommand(input, { tokens: true }); + const args = (data.tokens ?? []).filter((t) => t.class === "arg"); + expect(args.map((t) => input.slice(t.start, t.end))).toEqual(["address="]); + // Every claimed byte is in the FIRST statement. + const secondStart = input.indexOf("\n") + 1; + for (const t of args) expect(t.end).toBeLessThanOrEqual(secondStart); + // The refused statement's `gateway=` bytes stay unclassified — not `arg`. + const gatewayEq = input.indexOf("gateway="); + const atGateway = (data.tokens ?? []).find( + (t) => t.start <= gatewayEq && t.end > gatewayEq, + ); + expect(atGateway?.class).toBe("unclassified"); + // e11 is cited because SOME statement produced arg tokens. + expect(data.evidence.some((e) => e.id === "e11")).toBe(true); + }); +});