From 4f4d93c31d5bc35f2615bbc04c2e7a1309408310 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Wed, 12 Aug 2026 23:16:59 -0400 Subject: [PATCH 1/9] Measure term encoding slowdown by mechanism --- README.md | 141 ++-- benchmarking/reports/analysis.py | 476 ++++++++------ benchmarking/reports/catalog.py | 2 +- benchmarking/reports/interactive_runtime.py | 4 +- benchmarking/reports/presentation.py | 319 ++++++--- benchmarking/reports/render.py | 10 +- benchmarking/reports/store.py | 22 +- .../core-relations/src/free_join/execute.rs | 5 +- egglog/egglog-bridge/src/lib.rs | 13 +- egglog/egglog-reports/src/lib.rs | 292 ++++----- egglog/src/cli.rs | 10 +- egglog/src/lib.rs | 302 ++++++++- egglog/src/phase_timers.rs | 74 +++ egglog/src/proofs/proof_encoding_helpers.rs | 2 +- egglog/src/scheduler.rs | 1 + egglog/tests/integration_test.rs | 2 +- egglog/tests/timing_summary_cli.rs | 166 ++++- encoding-architecture-bridge.md | 577 ++++++++++++++++ incremental-unification-pr-roadmap.md | 615 ++++++++++++++++++ term-encoding-overhead-breakdown.md | 577 ++++++++++++++++ term-encoding-unification.md | 477 ++++++++++++++ .../__snapshots__/test_report_rendering.ambr | 352 +++++----- tests/report_fixtures.py | 57 +- tests/test_collection.py | 26 +- tests/test_report_analysis.py | 400 ++++++++++-- tests/test_report_rendering.py | 191 ++++-- tests/test_report_store.py | 13 +- 27 files changed, 4204 insertions(+), 922 deletions(-) create mode 100644 egglog/src/phase_timers.rs create mode 100644 encoding-architecture-bridge.md create mode 100644 incremental-unification-pr-roadmap.md create mode 100644 term-encoding-overhead-breakdown.md create mode 100644 term-encoding-unification.md diff --git a/README.md b/README.md index f76cdc0e..17631ff1 100644 --- a/README.md +++ b/README.md @@ -218,8 +218,8 @@ path. | --- | --- | | `summary` | comparison selection and headline summary | | `files` | per-file wall time and peak RSS estimates | -| `phases` | per-file phase confidence intervals, wall shares, and change contributions | -| `rulesets` | the top 10 changed rulesets per file, including phase deltas | +| `phases` | one additive slowdown decomposition across files and mechanisms | +| `rulesets` | Program/Equality driver groups and changed rulesets per file | The default is `summary`. For example: @@ -272,46 +272,95 @@ Every successful benchmark observation records timing from the same measured process. Timing collection is always enabled; requesting a detailed report does not rerun a diagnostic process or change the cache key. -The engine records these components per ruleset and the JSONL stores their raw -nanosecond totals: +The JSONL stores one sorted list of exclusive timing leaves. Each leaf has a +segmented `path` and a raw nanosecond total; parent totals are never stored. +Segmented paths keep a ruleset name such as `rules/λ` or one containing `/` +unambiguous. +Ruleset leaves have one of these shapes: + +- `program//` for source-origin ruleset work; +- `equality//` for encoded equality-maintenance work; +- `equality/rebuild/` for native Rebuild tails from source rulesets as + well as rebuild tails from maintenance rulesets. + +Rulesets receive an explicit semantic role when declared. Generated equality +maintenance is therefore not inferred from an `@` name prefix. Moving both +native and encoded implementations under `equality` makes their net cost an +ordinary candidate-minus-baseline difference. + +The recorded ruleset phases are: + +- Ruleset assembly: lazy cached-plan creation and per-invocation executable + ruleset construction. - Search: matching and join execution. - Apply: executing rule-head instructions and staging updates. -- Unattributed: measured pre-merge work that cannot be accurately classified - as Search or Apply. +- Execution: measured pre-merge work that cannot be accurately classified as + Search or Apply. - Merge: resolving and installing staged updates. - Rebuild: rebuilding indexes and e-graph state. -The engine measures one contiguous pre-merge interval and records the remainder -after Search and Apply as Unattributed. - -The phase report aggregates all rulesets and keeps two kinds of otherwise -hidden time distinct: - -- Execution overhead is the stored Unattributed component: measured work - inside ruleset execution that cannot be split accurately into Search or - Apply. -- Outside recorded rulesets is derived as process wall time minus Search, - Apply, Execution overhead, Merge, and Rebuild. It includes process setup, - reporting, teardown, and other work outside timed ruleset execution. - -Each file gets its own phase table. For both endpoints, it displays the phase -mean's 95% confidence interval and the phase's share of that endpoint's wall -time. It also displays the signed mean change and that phase's contribution to -the file's total wall-time change. Contributions may be negative or exceed -100% when phases offset each other. A negative Outside recorded rulesets value -is prefixed with `!`; it means recorded phase totals exceed wall time and should -be treated as an attribution warning. - -The ruleset report totals all five stored components for each ruleset, aligns -the union of names across the two endpoints, and ranks by the absolute -candidate-minus-baseline total difference. It omits exact-zero changes and -displays at most 10 rulesets per file. Each row includes the baseline and -candidate total confidence intervals, total change, and descriptive Search, -Apply, Execution overhead, Merge, and Rebuild changes. Timings are aggregated -across the selected observations; iterations are not separate report rows. A -ruleset absent from one endpoint is displayed as `—`, while a measured zero -remains `0 ns`. +The engine measures one contiguous pre-merge interval, including per-run setup, +and records the remainder after Search and Apply as Execution. Assembly done +inside native rebuild remains part of Rebuild rather than being counted twice. +All six leaves are retained for every invoked ruleset, including zero values. + +The same list contains these process leaves outside ordinary ruleset execution: + +- `typecheck/total`: total source and generated typechecking, including source + checking performed by the encoded mode's cloned checker; +- `frontend/parse`, `frontend/other`, and `frontend/install`: parsing, other + lowering work, and post-resolution execution of declarations; +- `commands/actions`, `commands/check`, and `commands/other`: actions/input, + complete check evaluation, and other commands or schedule driving. + +Check queries are transient backend rules in both encoded and unencoded modes. +Their backend execution and surrounding compilation/validation overhead are +charged together to `commands/check`; they do not appear as program rulesets. +One known command boundary remains: if a top-level action such as `(union ...)` +causes `flush_updates` to rebuild, that rebuild stays in `commands/actions`. +Top-level actions are timed as commands and their transient backend report is +not inserted into the named-ruleset ledger, so this work does not appear under +Equality. + +Residual is derived per observation as external wall time minus every recorded +leaf. It includes process setup, reporting, teardown, and any still- +uninstrumented work. + +At `--detail phases`, one additive slowdown-decomposition table has a suite row +and one row per file. Its rendered headers are `Wall Δ`, `Typecheck`, +`Frontend`, `Program`, `Equality`, `Commands`, and `Residual`. Every mechanism +cell displays its share of the wall-time change first, then +candidate-minus-baseline milliseconds. `◆` marks the largest absolute +mechanism share in each row; Rich and interactive reports also bold that cell, +dim contributions below 5%, and color improvements green. Expected overhead is +neutral rather than red; warning and error colors are reserved for suspect +measurements. Percentages may be negative or exceed 100% when mechanisms +offset. `!` on a Residual cell means at least one endpoint's mean recorded total +exceeded its wall time. + +At `--detail rulesets`, one compact driver table appears per file. Its +`Program rules — own work` and `Equality/rebuild — net` parent rows exactly +match the corresponding cells in the decomposition, show their wall share, +and report what fraction of the file's wall-time change they jointly account +for. Child rows use a `↳` prefix in Rich, Markdown, and interactive reports. + +Program children contain only source-rule Assembly, Search, Apply, Execution, +and Merge. They never inherit the native Rebuild tail that happened to follow +their invocation. At most five changed source rulesets are ranked by absolute +own-work difference; `Other (N more source rulesets)` is the exact additive +sum of the omitted source children. Equality children contain every changed +encoded-maintenance ruleset and, when nonzero, one global +`Native rebuild replaced` row. Thus the children beneath each parent add +exactly to that parent without a cross-mechanism reconciliation convention. +Zero children are omitted. + +Only parent rows show wall share. Every row retains a compact phase summary. +That summary includes every phase whose absolute change is at least +`max(1 ms, 10% of |row change|)`, always includes the dominant phase marked +with `◆`, and uses fixed Assembly, Search, Apply, Execution, Merge, Rebuild +order. `…` means smaller nonzero phase changes were omitted from display, not +from accounting. Benchmarks run single-threaded. This keeps Search and Apply attribution additive for egglog's interleaved executor. @@ -447,9 +496,11 @@ Each observation contains target and workload provenance, exact cache coordinates, status, wall time, peak RSS, and failure details. A top-level report schema version covers both the persisted shape and measurement semantics, so methodology changes cannot silently reuse stale -measurements. Successful observations also contain the version-2 per-ruleset -timing summary: name plus Search, Apply, Unattributed, Merge, and Rebuild -nanoseconds. +measurements. Successful observations also contain the version-3 timing +summary: one open list of exclusive `{path: [segment, ...], ns: value}` leaves. +Adding detail below an existing responsibility prefix does not require another +parallel record shape. Changes to timing coverage or meaning still require a +schema-version change so stale measurements cannot be reused silently. Timed-out rows have null wall time, peak RSS, and timing summary. Failed rows have no timing summary and retain whatever process measurements the operating @@ -461,13 +512,15 @@ and timing-summary schema versions and requires successful rows to contain timing data. It trusts the tool's typed writer rather than repeating the `TypedDict` as runtime field-by-field validation. A schema change intentionally invalidates existing caches: move or remove an incompatible report and recompute -it. +it. Analysis invariants use ordinary exceptions rather than `assert`, so +optimized Python does not silently accept a persisted Residual leaf or an +unknown top-level timing responsibility. ### Report-analysis ownership `ComparisonSpec` owns the exact endpoints, files, rounds, and timeout; `store.py` owns physical row order and cache selection. `analysis.py` computes -immutable summary, file, phase, and ruleset rows, while `presentation.py` maps +immutable summary, file, mechanism-decomposition, and ruleset rows, while `presentation.py` maps them to the renderer-neutral catalog. Rich, Markdown, and the interactive page serialize that catalog without recomputing report facts. @@ -504,9 +557,9 @@ shown. No median or geometric mean is mixed into this minimal headline. A timed-out, failed, or otherwise incomplete selected result invalidates the suite result that depends on it. Valid per-file tail comparisons remain useful -when an unrelated file is incomplete. Phase endpoint means and ruleset totals -receive confidence intervals; phase contributions and individual ruleset -component deltas are descriptive diagnostics. +when an unrelated file is incomplete. Mechanism contributions and individual +ruleset component deltas are descriptive diagnostics; ruleset totals receive +confidence intervals. The `<2x` proof goal is established only when the upper bound of the suite wall ratio's 95% confidence interval is below `2x` for a proofs-versus-off diff --git a/benchmarking/reports/analysis.py b/benchmarking/reports/analysis.py index e237d62a..c8553147 100644 --- a/benchmarking/reports/analysis.py +++ b/benchmarking/reports/analysis.py @@ -10,9 +10,7 @@ import math import statistics -from collections.abc import Iterable -from dataclasses import dataclass -from typing import Literal, NamedTuple +from typing import Literal, NamedTuple, cast from scipy import stats @@ -22,15 +20,34 @@ MetricName = Literal["wall_sec", "max_rss_bytes"] ResultClass = Literal["higher", "invalid", "lower", "point_only", "unclear"] SummaryKind = Literal["suite", "lowest_file", "highest_file"] -PhaseName = Literal["search", "apply", "unattributed", "merge", "rebuild", "outside"] -RulesetPhaseName = Literal["search", "apply", "unattributed", "merge", "rebuild"] +MechanismName = Literal["typecheck", "frontend", "program", "equality", "commands", "residual"] +RulesetPhaseName = Literal["assembly", "search", "apply", "execution", "merge", "rebuild"] +RulesetMechanism = Literal["program", "equality"] +RulesetRowKind = Literal["aggregate", "ruleset", "native_rebuild", "other"] type _MetricKey = tuple[int, int, MetricName] type _ObservationKey = tuple[int, int] +type _TimingPath = tuple[str, ...] _METRICS: tuple[MetricName, ...] = ("wall_sec", "max_rss_bytes") -_RULESET_PHASES: tuple[RulesetPhaseName, ...] = ("search", "apply", "unattributed", "merge", "rebuild") -_PHASES: tuple[PhaseName, ...] = (*_RULESET_PHASES, "outside") +_RULESET_PHASES: tuple[RulesetPhaseName, ...] = ( + "assembly", + "search", + "apply", + "execution", + "merge", + "rebuild", +) +_RULESET_MECHANISMS: tuple[RulesetMechanism, ...] = ("program", "equality") +_MECHANISMS: tuple[MechanismName, ...] = ( + "typecheck", + "frontend", + "program", + "equality", + "commands", + "residual", +) +RULESET_CONTRIBUTOR_LIMIT = 5 class Estimate(NamedTuple): @@ -49,40 +66,19 @@ class RatioEstimate(NamedTuple): issue: str | None -class PhaseEstimate(NamedTuple): - """One phase estimate and its share of endpoint wall time.""" - - timing: Estimate - wall_share: float | None - - class PhaseValues(NamedTuple): - """Five recorded timing components aggregated for one observation/ruleset.""" + """Six recorded timing components aggregated for one observation/ruleset.""" + assembly: float search: float apply: float - unattributed: float + execution: float merge: float rebuild: float - @property - def total(self) -> float: - return sum(self) - - def phase(self, name: RulesetPhaseName) -> float: - if name == "search": - return self.search - if name == "apply": - return self.apply - if name == "unattributed": - return self.unattributed - if name == "merge": - return self.merge - return self.rebuild - class RulesetDelta(NamedTuple): - """One exact total delta and its five timing-component deltas.""" + """One exact total delta and its six timing-component deltas.""" total: float phases: PhaseValues @@ -107,25 +103,42 @@ class FileComparisonView(NamedTuple): ratio: RatioEstimate -class PhaseComparisonView(NamedTuple): - """One exhaustive per-file wall-time phase comparison.""" +class SlowdownCell(NamedTuple): + """One mechanism's delta and share of the observed wall slowdown.""" - file_order: int - phase: PhaseName - baseline: PhaseEstimate - candidate: PhaseEstimate delta_ns: float | None - wall_delta_contribution: float | None + slowdown_share: float | None + + +class SlowdownValues(NamedTuple): + """The six additive mechanism cells displayed for one row.""" + + typecheck: SlowdownCell + frontend: SlowdownCell + program: SlowdownCell + equality: SlowdownCell + commands: SlowdownCell + residual: SlowdownCell -class RulesetComparisonView(NamedTuple): - """One top absolute-total-delta ruleset with component deltas.""" +class SlowdownDecompositionView(NamedTuple): + """One per-file or suite-wide additive slowdown decomposition.""" + + file_order: int | None + wall_delta_ns: float | None + mechanisms: SlowdownValues + residual_warning: bool + issue: str | None + + +class RulesetContributorView(NamedTuple): + """One mechanism parent, named child, native rebuild, or exact remainder.""" file_order: int - ruleset_count: int + kind: RulesetRowKind + mechanism: RulesetMechanism name: str - baseline: Estimate | None - candidate: Estimate | None + ruleset_count: int delta: RulesetDelta @@ -134,8 +147,8 @@ class PairReportViewData(NamedTuple): summary: tuple[SummaryView, ...] files: tuple[FileComparisonView, ...] - phases: tuple[PhaseComparisonView, ...] - rulesets: tuple[RulesetComparisonView, ...] + decomposition: tuple[SlowdownDecompositionView, ...] + rulesets: tuple[RulesetContributorView, ...] class _MetricEstimate(NamedTuple): @@ -145,30 +158,12 @@ class _MetricEstimate(NamedTuple): issue: str | None -@dataclass -class _RulesetSamples: - """Sparse per-observation samples; omitted observations contribute zero.""" - - total: list[float] - phases: dict[RulesetPhaseName, list[float]] - - -@dataclass -class _TimingAggregate: - """One-pass phase and ruleset samples for an endpoint/file selection.""" +class _TimingAggregate(NamedTuple): + """Aligned samples for the open timing paths in one endpoint/file selection.""" observation_count: int - phases: dict[PhaseName, list[float]] - rulesets: dict[str, _RulesetSamples] - - -class _RankedRuleset(NamedTuple): - """One changed ruleset before the per-file top-ten cutoff.""" - - name: str - baseline: _MetricEstimate | None - candidate: _MetricEstimate | None - delta: RulesetDelta + paths: dict[_TimingPath, list[float]] + residuals: list[float] def analyze_pair( @@ -191,11 +186,11 @@ def analyze_pair( return PairReportViewData(summary, file_rows, (), ()) timing = _timing_aggregates(observations) - phases = _phase_comparisons(comparison, timing, issues, estimates, t_critical) + decomposition = _slowdown_decomposition(comparison, timing, issues, estimates) if detail == "phases": - return PairReportViewData(summary, file_rows, phases, ()) - rulesets = _ruleset_comparisons(comparison, timing, issues, t_critical) - return PairReportViewData(summary, file_rows, phases, rulesets) + return PairReportViewData(summary, file_rows, decomposition, ()) + rulesets = _ruleset_contributors(comparison, timing, issues) + return PairReportViewData(summary, file_rows, decomposition, rulesets) def _selected_observations( @@ -267,20 +262,18 @@ def _ratio_estimate( baseline_mean = baseline.estimate.point candidate_mean = candidate.estimate.point issue = baseline.issue or candidate.issue - if issue is None and (baseline_mean is None or candidate_mean is None): - issue = "estimate unavailable" - if issue is None and baseline_mean is not None and baseline_mean <= 0: - issue = "baseline mean is not positive" if issue is not None: return RatioEstimate(Estimate(None, None, None), "invalid", issue) + if baseline_mean is None or candidate_mean is None: + return RatioEstimate(Estimate(None, None, None), "invalid", "estimate unavailable") + if baseline_mean <= 0: + return RatioEstimate(Estimate(None, None, None), "invalid", "baseline mean is not positive") - assert baseline_mean is not None and candidate_mean is not None point = candidate_mean / baseline_mean if min(baseline.sample_count, candidate.sample_count) < 2: return RatioEstimate(Estimate(point, None, None), "point_only", "CI undefined for n < 2") - assert baseline.var_mean is not None - assert candidate.var_mean is not None - assert t_critical is not None + if baseline.var_mean is None or candidate.var_mean is None or t_critical is None: + raise ValueError("multi-sample ratio is missing variance or its t critical value") critical_squared = t_critical * t_critical fieller_a = baseline_mean * baseline_mean - critical_squared * baseline.var_mean fieller_d = candidate_mean * candidate_mean - critical_squared * candidate.var_mean @@ -370,26 +363,25 @@ def _summary_rows( return tuple(rows) -def _phase_comparisons( +def _slowdown_decomposition( comparison: ComparisonSpec, timing: dict[_ObservationKey, _TimingAggregate], issues: dict[_ObservationKey, str | None], metric_estimates: dict[_MetricKey, _MetricEstimate], - t_critical: float | None, -) -> tuple[PhaseComparisonView, ...]: - estimates: dict[tuple[int, int, PhaseName], _MetricEstimate] = {} +) -> tuple[SlowdownDecompositionView, ...]: + points: dict[tuple[int, int, MechanismName], float | None] = {} for (endpoint_order, file_order), aggregate in timing.items(): - for phase in _PHASES: + for mechanism in _MECHANISMS: issue = issues[(endpoint_order, file_order)] - if phase == "outside" and issue is None: + if mechanism == "residual" and issue is None: issue = metric_estimates[(endpoint_order, file_order, "wall_sec")].issue - estimates[(endpoint_order, file_order, phase)] = _sample_estimate( - aggregate.phases[phase], - issue, - t_critical, + paths = [path for path in aggregate.paths if path[0] == mechanism] + values = aggregate.residuals if mechanism == "residual" else _sum_path_samples(aggregate, paths) + points[(endpoint_order, file_order, mechanism)] = ( + statistics.fmean(values) if issue is None and values else None ) - result: list[PhaseComparisonView] = [] + result: list[SlowdownDecompositionView] = [] for file_order in range(len(comparison.files)): baseline_wall = metric_estimates[(0, file_order, "wall_sec")].estimate.point candidate_wall = metric_estimates[(1, file_order, "wall_sec")].estimate.point @@ -398,29 +390,49 @@ def _phase_comparisons( if baseline_wall is None or candidate_wall is None else (candidate_wall - baseline_wall) * 1_000_000_000.0 ) - for phase in _PHASES: - baseline = estimates[(0, file_order, phase)] - candidate = estimates[(1, file_order, phase)] - baseline_point = baseline.estimate.point - candidate_point = candidate.estimate.point + cells: list[SlowdownCell] = [] + for mechanism in _MECHANISMS: + baseline_point = points[(0, file_order, mechanism)] + candidate_point = points[(1, file_order, mechanism)] delta = None if baseline_point is None or candidate_point is None else candidate_point - baseline_point - result.append( - PhaseComparisonView( - file_order, - phase, - PhaseEstimate( - baseline.estimate, - _share(baseline_point, baseline_wall, scale=1_000_000_000.0), - ), - PhaseEstimate( - candidate.estimate, - _share(candidate_point, candidate_wall, scale=1_000_000_000.0), - ), - delta, - _share(delta, wall_delta_ns), - ) + cells.append(SlowdownCell(delta, _share(delta, wall_delta_ns))) + baseline_residual = points[(0, file_order, "residual")] + candidate_residual = points[(1, file_order, "residual")] + issue = ( + issues[(0, file_order)] + or issues[(1, file_order)] + or metric_estimates[(0, file_order, "wall_sec")].issue + or metric_estimates[(1, file_order, "wall_sec")].issue + ) + result.append( + SlowdownDecompositionView( + file_order, + wall_delta_ns, + SlowdownValues(*cells), + (baseline_residual is not None and baseline_residual < 0) + or (candidate_residual is not None and candidate_residual < 0), + issue, ) - return tuple(result) + ) + + suite_issue = next((row.issue for row in result if row.issue is not None), None) + if suite_issue is None: + suite_wall_delta = sum(cast(float, row.wall_delta_ns) for row in result) + suite_cells = [] + for mechanism_index in range(len(_MECHANISMS)): + delta = sum(cast(float, row.mechanisms[mechanism_index].delta_ns) for row in result) + suite_cells.append(SlowdownCell(delta, _share(delta, suite_wall_delta))) + else: + suite_wall_delta = None + suite_cells = [SlowdownCell(None, None) for _ in _MECHANISMS] + suite = SlowdownDecompositionView( + None, + suite_wall_delta, + SlowdownValues(*suite_cells), + any(row.residual_warning for row in result), + suite_issue, + ) + return (suite, *result) def _timing_aggregates( @@ -428,131 +440,190 @@ def _timing_aggregates( ) -> dict[_ObservationKey, _TimingAggregate]: result: dict[_ObservationKey, _TimingAggregate] = {} for key, rows in observations.items(): - aggregate = _TimingAggregate( - len(rows), - {phase: [] for phase in _PHASES}, - {}, - ) - for row in rows: + aggregate = _TimingAggregate(len(rows), {}, []) + for observation_index, row in enumerate(rows): record = row.record if record["status"] != "success": + for samples in aggregate.paths.values(): + samples.append(0.0) continue summary = record["timing_summary"] - assert summary is not None - per_ruleset: dict[str, PhaseValues] = {} - for ruleset in summary["rulesets"]: - totals = PhaseValues( - float(ruleset["search_ns"]), - float(ruleset["apply_ns"]), - float(ruleset["unattributed_ns"]), - float(ruleset["merge_ns"]), - float(ruleset["rebuild_ns"]), - ) - per_ruleset[ruleset["name"]] = _add_totals(per_ruleset.get(ruleset["name"], _ZERO_PHASE_TOTALS), totals) - recorded = _sum_totals(per_ruleset.values()) - for phase in _RULESET_PHASES: - aggregate.phases[phase].append(recorded.phase(phase)) + if summary is None: + raise ValueError("successful benchmark record is missing its timing summary") + observation: dict[_TimingPath, float] = {} + recorded = 0.0 + for leaf in summary["timings"]: + path = tuple(leaf["path"]) + if not path: + raise ValueError("timing path must not be empty") + if path[0] == "residual": + raise ValueError("residual is derived rather than recorded") + if path[0] not in _MECHANISMS[:-1]: + raise ValueError(f"unknown timing responsibility {path[0]!r}") + duration = float(leaf["ns"]) + observation[path] = observation.get(path, 0.0) + duration + recorded += duration + for path, samples in aggregate.paths.items(): + samples.append(observation.pop(path, 0.0)) + for path, duration in observation.items(): + aggregate.paths[path] = [0.0] * observation_index + [duration] wall_sec = record["wall_sec"] if wall_sec is not None: - aggregate.phases["outside"].append(wall_sec * 1_000_000_000.0 - recorded.total) - for name, totals in per_ruleset.items(): - samples = aggregate.rulesets.setdefault( - name, - _RulesetSamples([], {phase: [] for phase in _RULESET_PHASES}), - ) - samples.total.append(totals.total) - for phase in _RULESET_PHASES: - samples.phases[phase].append(totals.phase(phase)) + aggregate.residuals.append(wall_sec * 1_000_000_000.0 - recorded) result[key] = aggregate return result -_ZERO_PHASE_TOTALS = PhaseValues(0.0, 0.0, 0.0, 0.0, 0.0) - - -def _add_totals(left: PhaseValues, right: PhaseValues) -> PhaseValues: - return PhaseValues( - left.search + right.search, - left.apply + right.apply, - left.unattributed + right.unattributed, - left.merge + right.merge, - left.rebuild + right.rebuild, - ) - +def _sum_path_samples(aggregate: _TimingAggregate, paths: list[_TimingPath]) -> list[float]: + """Add selected exclusive leaves observation by observation.""" -def _sum_totals(values: Iterable[PhaseValues]) -> PhaseValues: - result = _ZERO_PHASE_TOTALS - for value in values: - result = _add_totals(result, value) - return result + return [math.fsum(aggregate.paths[path][index] for path in paths) for index in range(aggregate.observation_count)] -def _ruleset_comparisons( +def _ruleset_contributors( comparison: ComparisonSpec, timing: dict[_ObservationKey, _TimingAggregate], issues: dict[_ObservationKey, str | None], - t_critical: float | None, -) -> tuple[RulesetComparisonView, ...]: - result: list[RulesetComparisonView] = [] +) -> tuple[RulesetContributorView, ...]: + """Unfold Program and Equality into truthful per-file child partitions.""" + + result: list[RulesetContributorView] = [] for file_order in range(len(comparison.files)): if issues[(0, file_order)] is not None or issues[(1, file_order)] is not None: continue - names = sorted({name for endpoint_order in (0, 1) for name in timing[(endpoint_order, file_order)].rulesets}) - comparisons: list[_RankedRuleset] = [] - for name in names: - baseline = _ruleset_estimate(timing[(0, file_order)], name, None, t_critical) - candidate = _ruleset_estimate(timing[(1, file_order)], name, None, t_critical) - total_delta = _estimate_point(candidate) - _estimate_point(baseline) - if total_delta == 0.0: - continue - delta = RulesetDelta(total_delta, _ruleset_phase_deltas(timing, file_order, name, t_critical)) - comparisons.append(_RankedRuleset(name, baseline, candidate, delta)) - comparisons.sort(key=lambda row: (-abs(row.delta.total), row.name)) - count = len(comparisons) - for row in comparisons[:10]: + source_names = sorted( + { + path[2] + for endpoint_order in (0, 1) + for path in timing[(endpoint_order, file_order)].paths + if len(path) == 3 and path[0] == "program" + } + ) + maintenance_names = sorted( + { + path[2] + for endpoint_order in (0, 1) + for path in timing[(endpoint_order, file_order)].paths + if len(path) == 3 and path[0] == "equality" and path[1] != "rebuild" + } + ) + + names_by_mechanism = {"program": source_names, "equality": maintenance_names} + children: dict[RulesetMechanism, list[RulesetContributorView]] = {"program": [], "equality": []} + for mechanism in _RULESET_MECHANISMS: + for name in names_by_mechanism[mechanism]: + delta = _ruleset_phase_deltas(timing, file_order, name, mechanism) + if any(delta.phases): + children[mechanism].append(RulesetContributorView(file_order, "ruleset", mechanism, name, 1, delta)) + children[mechanism].sort(key=lambda row: (-abs(row.delta.total), row.name)) + + source_rebuild_deltas: list[float] = [ + rebuild_delta + for name in source_names + if ( + rebuild_delta := _ruleset_phase_delta( + timing, + file_order, + name, + "equality", + "rebuild", + ) + ) + != 0 + ] + if source_rebuild_deltas: + rebuild_delta = math.fsum(source_rebuild_deltas) + children["equality"].append( + RulesetContributorView( + file_order, + "native_rebuild", + "equality", + "", + 0, + RulesetDelta( + rebuild_delta, + PhaseValues(0.0, 0.0, 0.0, 0.0, 0.0, rebuild_delta), + ), + ) + ) + + for mechanism in _RULESET_MECHANISMS: + group = children[mechanism] result.append( - RulesetComparisonView( + RulesetContributorView( file_order, - count, - row.name, - None if row.baseline is None else row.baseline.estimate, - None if row.candidate is None else row.candidate.estimate, - row.delta, + "aggregate", + mechanism, + "", + sum(row.ruleset_count for row in group), + _sum_ruleset_deltas(group), ) ) + if mechanism == "program" and len(group) > RULESET_CONTRIBUTOR_LIMIT: + omitted = group[RULESET_CONTRIBUTOR_LIMIT:] + result.extend(group[:RULESET_CONTRIBUTOR_LIMIT]) + result.append( + RulesetContributorView( + file_order, + "other", + mechanism, + "", + len(omitted), + _sum_ruleset_deltas(omitted), + ) + ) + else: + result.extend(group) return tuple(result) -def _ruleset_estimate( - aggregate: _TimingAggregate, +def _ruleset_phase_delta( + timing: dict[_ObservationKey, _TimingAggregate], + file_order: int, name: str, - phase: RulesetPhaseName | None, - t_critical: float | None, -) -> _MetricEstimate | None: - samples = aggregate.rulesets.get(name) - if samples is None: - return None - observed = samples.total if phase is None else samples.phases[phase] - values = [*observed, *(0.0 for _ in range(aggregate.observation_count - len(observed)))] - return _sample_estimate(values, None, t_critical) + responsibility: RulesetMechanism, + phase: RulesetPhaseName, +) -> float: + """Subtract one named responsibility/phase mean across the endpoints.""" + def mean(endpoint_order: int) -> float: + aggregate = timing[(endpoint_order, file_order)] + paths: list[_TimingPath] = [ + path + for path in aggregate.paths + if len(path) == 3 and path[0] == responsibility and path[1] == phase and path[2] == name + ] + if not paths: + return 0.0 + return statistics.fmean(_sum_path_samples(aggregate, paths)) -def _estimate_point(estimate: _MetricEstimate | None) -> float: - return 0.0 if estimate is None or estimate.estimate.point is None else estimate.estimate.point + return mean(1) - mean(0) def _ruleset_phase_deltas( timing: dict[_ObservationKey, _TimingAggregate], file_order: int, name: str, - t_critical: float | None, -) -> PhaseValues: - def delta(phase: RulesetPhaseName) -> float: - candidate = _ruleset_estimate(timing[(1, file_order)], name, phase, t_critical) - baseline = _ruleset_estimate(timing[(0, file_order)], name, phase, t_critical) - return _estimate_point(candidate) - _estimate_point(baseline) + responsibility: RulesetMechanism, +) -> RulesetDelta: + """Return own-work Program phases or complete Equality-maintenance phases.""" + + phases = PhaseValues( + *( + 0.0 + if responsibility == "program" and phase == "rebuild" + else _ruleset_phase_delta(timing, file_order, name, responsibility, phase) + for phase in _RULESET_PHASES + ) + ) + return RulesetDelta(math.fsum(phases), phases) + + +def _sum_ruleset_deltas(rows: list[RulesetContributorView]) -> RulesetDelta: + """Sum a ruleset partition without losing phase-level additivity.""" - return PhaseValues(delta("search"), delta("apply"), delta("unattributed"), delta("merge"), delta("rebuild")) + phases = PhaseValues(*(math.fsum(row.delta.phases[index] for row in rows) for index in range(len(_RULESET_PHASES)))) + return RulesetDelta(math.fsum(row.delta.total for row in rows), phases) def _sample_estimate( @@ -566,7 +637,8 @@ def _sample_estimate( ci_high: float | None = None if mean is not None and len(values) >= 2: var_mean = statistics.variance(values) / len(values) - assert t_critical is not None + if t_critical is None: + raise ValueError("multi-sample estimate is missing its t critical value") half_width = t_critical * math.sqrt(var_mean) ci_low = mean - half_width ci_high = mean + half_width diff --git a/benchmarking/reports/catalog.py b/benchmarking/reports/catalog.py index 1b095759..3a824008 100644 --- a/benchmarking/reports/catalog.py +++ b/benchmarking/reports/catalog.py @@ -14,7 +14,7 @@ type ReportScalar = str | int | float | bool | None TableAlignment = Literal["left", "right"] -CellTone = Literal["default", "positive", "negative", "warning", "error", "muted"] +CellTone = Literal["default", "positive", "emphasis", "warning", "error", "muted"] @dataclass(frozen=True) diff --git a/benchmarking/reports/interactive_runtime.py b/benchmarking/reports/interactive_runtime.py index 5c1be58f..344aa6b9 100644 --- a/benchmarking/reports/interactive_runtime.py +++ b/benchmarking/reports/interactive_runtime.py @@ -429,8 +429,8 @@ def _primitive_display(value: JsonScalar) -> str: def _tone_style(tone: CellTone) -> dict[str, JsonValue]: if tone == "positive": return {"color": "green"} - if tone == "negative": - return {"color": "red"} + if tone == "emphasis": + return {"bold": True} if tone == "warning": return {"color": "yellow"} if tone == "error": diff --git a/benchmarking/reports/presentation.py b/benchmarking/reports/presentation.py index c8db27d0..7cd9c595 100644 --- a/benchmarking/reports/presentation.py +++ b/benchmarking/reports/presentation.py @@ -1,7 +1,7 @@ """Build the canonical benchmark presentation and format its values. This module maps typed statistics from :mod:`benchmarking.reports.analysis` -into Comparison, Summary, Files, Phases, and Rulesets sections. It owns shared +into Comparison, Summary, Files, Mechanisms, and Rulesets sections. It owns shared labels, units, interval formatting, and result wording; Rich and Markdown only serialize the resulting catalog. """ @@ -14,16 +14,17 @@ from ..models import BenchmarkEndpoint, ComparisonSpec, DetailLevel, FileSpec from .analysis import ( + RULESET_CONTRIBUTOR_LIMIT, Estimate, FileComparisonView, MetricName, PairReportViewData, - PhaseComparisonView, - PhaseEstimate, - PhaseName, RatioEstimate, ResultClass, - RulesetComparisonView, + RulesetContributorView, + RulesetDelta, + SlowdownCell, + SlowdownDecompositionView, SummaryView, analyze_pair, ) @@ -52,27 +53,26 @@ "rulesets": 3, } RATIO_DIRECTION = "Ratios are candidate / baseline; below 1 is lower and above 1 is higher." -PHASE_CAPTION = ( - "Endpoint cells show a 95% CI (or one-round point) and that phase's share of endpoint wall time. " - "Delta is the signed candidate − baseline mean; Δ contribution is the phase's share of the wall-time " - "change and may be negative or exceed 100% when phases offset. Execution overhead is stored per-ruleset " - "unattributed time. Outside recorded rulesets is wall time minus all five recorded phases; ! marks a negative " - "residual." +DECOMPOSITION_CAPTION = ( + "Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. " + "Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every " + "phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with " + "native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or " + "exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below " + "5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same " + "information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean " + "residual is negative." ) RULESET_CAPTION = ( - "Totals show a 95% CI or one-round point. S/A/Exec/M/R are signed candidate − baseline mean deltas for " - "Search, Apply, Execution overhead (stored unattributed time), Merge, and Rebuild." + "Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those " + "cells and alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, " + "and Merge; Equality children contain every encoded maintenance ruleset plus one global Native rebuild replaced " + "row. ↳ marks children in every format. Zero children are hidden. Source children are ranked by absolute own-work " + f"Δ (top {RULESET_CONTRIBUTOR_LIMIT} plus an exact per-group Other); every nonzero maintenance child is shown. " + "Important phases include every |phase Δ| ≥ max(1 ms, 10% of |row Δ|), always include the dominant phase (◆), " + "and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases." ) -PHASE_LABELS: dict[PhaseName, str] = { - "search": "Search", - "apply": "Apply", - "unattributed": "Execution overhead", - "merge": "Merge", - "rebuild": "Rebuild", - "outside": "Outside recorded rulesets", -} - def build_report_catalog( store: ReportStore, @@ -91,7 +91,7 @@ def build_report_catalog( if _includes(detail, "files"): sections.append(_files_section(views.files, comparison, file_labels)) if _includes(detail, "phases"): - sections.append(_phases_section(views.phases, comparison, file_labels)) + sections.append(_phases_section(views.decomposition, comparison, file_labels)) if _includes(detail, "rulesets"): sections.append(_rulesets_section(views, comparison, file_labels)) return ReportCatalog(tuple(sections)) @@ -305,39 +305,98 @@ def _files_section( def _phases_section( - rows: Sequence[PhaseComparisonView], + rows: Sequence[SlowdownDecompositionView], comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: - by_file: dict[int, list[PhaseComparisonView]] = {} + report_rows = [] for row in rows: - by_file.setdefault(row.file_order, []).append(row) - blocks: list[ReportBlock] = [ - ReportMessage(report_id("message", "phases", "guide"), None, PHASE_CAPTION, tone="muted") - ] - for file_order, file in enumerate(comparison.files): - blocks.append( - _table( - report_id("table", "phases", file.sha256, file.fact_directory_sha256), - f"Phase comparison — {file_labels[file]}", - ("phase", "baseline", "candidate", "delta", "wall_delta"), - ("Phase", "Baseline (95% CI · wall)", "Candidate (95% CI · wall)", "Delta", "Δ contribution"), - tuple(_phase_row(row, file) for row in by_file[file_order]), - alignments=("left", "right", "right", "right", "right"), + if row.file_order is None: + row_id = report_id("row", "phases", "suite") + label = "Suite total" + else: + file = comparison.files[row.file_order] + row_id = report_id("row", "phases", file.sha256, file.fact_directory_sha256) + label = file_labels[file] + comparable = [index for index, cell in enumerate(row.mechanisms) if cell.slowdown_share is not None] + leader = max(comparable, key=lambda index: abs(row.mechanisms[index].slowdown_share or 0.0), default=None) + if leader is not None and row.mechanisms[leader].slowdown_share == 0.0: + leader = None + mechanism_cells = tuple( + _slowdown_cell( + cell, + leader=index == leader, + warning=row.residual_warning and index == len(row.mechanisms) - 1, ) + for index, cell in enumerate(row.mechanisms) ) - return ReportSection("phases", "Phase comparison", tuple(blocks)) + report_rows.append( + _row( + row_id, + text_cell(row.file_order, label), + text_cell( + row.wall_delta_ns, + _format_delta_ms(row.wall_delta_ns), + tone=_delta_tone(row.wall_delta_ns), + ), + *mechanism_cells, + ) + ) + table = _table( + report_id("table", "phases", "decomposition"), + "Slowdown decomposition", + ("file", "wall_delta", "typecheck", "frontend", "program", "equality", "commands", "residual"), + ( + "File", + "Wall Δ", + "Typecheck", + "Frontend", + "Program", + "Equality", + "Commands", + "Residual", + ), + tuple(report_rows), + caption=DECOMPOSITION_CAPTION, + alignments=("left", "right", "right", "right", "right", "right", "right", "right"), + ) + return ReportSection("phases", "Slowdown decomposition", (table,)) + + +def _slowdown_cell(cell: SlowdownCell, *, leader: bool, warning: bool) -> ReportCell: + duration = _format_delta_ms(cell.delta_ns) + share = _format_percent(cell.slowdown_share, signed=True) + marker = "◆ " if leader else "" + display = NULL if cell.delta_ns is None else f"{marker}{share} {duration}" + if warning: + display = f"!{display}" + return text_cell( + cell.slowdown_share, + display, + tone=_delta_tone(cell.delta_ns, share=cell.slowdown_share, emphasis=leader, warning=warning), + ) -def _phase_row(row: PhaseComparisonView, file: FileSpec) -> ReportRow: - return _row( - report_id("row", "phases", file.sha256, file.fact_directory_sha256, row.phase), - text_cell(row.phase, PHASE_LABELS[row.phase]), - _phase_estimate_cell(row.baseline, attribution=row.phase == "outside"), - _phase_estimate_cell(row.candidate, attribution=row.phase == "outside"), - text_cell(row.delta_ns, format_duration(row.delta_ns, signed=True)), - text_cell(row.wall_delta_contribution, _format_percent(row.wall_delta_contribution, signed=True)), - ) +def _delta_tone( + delta_ns: float | None, + *, + share: float | None = None, + emphasis: bool = False, + warning: bool = False, +) -> CellTone: + """Apply the report's anomaly-first styling policy to one signed delta.""" + + if warning: + return "warning" + if emphasis: + return "emphasis" + if share is not None and abs(share) < 0.05: + return "muted" + if delta_ns is not None and delta_ns < 0: + return "positive" + if delta_ns == 0: + return "muted" + return "default" def _rulesets_section( @@ -345,7 +404,7 @@ def _rulesets_section( comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: - by_file: dict[int, list[RulesetComparisonView]] = {} + by_file: dict[int, list[RulesetContributorView]] = {} for row in views.rulesets: by_file.setdefault(row.file_order, []).append(row) file_issues = { @@ -353,11 +412,12 @@ def _rulesets_section( for row in views.files if row.metric == "wall_sec" and row.ratio.issue is not None } + wall_deltas = {row.file_order: row.wall_delta_ns for row in views.decomposition if row.file_order is not None} blocks: list[ReportBlock] = [] if views.rulesets: blocks.append(ReportMessage(report_id("message", "rulesets", "guide"), None, RULESET_CAPTION, tone="muted")) for file_order, file in enumerate(comparison.files): - title = f"Ruleset comparison — {file_labels[file]}" + title = f"Ruleset drivers — {file_labels[file]}" rulesets = by_file.get(file_order, []) if not rulesets: issue = file_issues.get(file_order) @@ -370,47 +430,101 @@ def _rulesets_section( ) ) continue - count = rulesets[0].ruleset_count - caption = None if count <= 10 else f"Showing 10 of {count} changed rulesets by absolute total delta." + wall_delta = wall_deltas.get(file_order) + parents = {row.mechanism: row for row in rulesets if row.kind == "aggregate"} + program_parent = parents["program"] + equality_parent = parents["equality"] + coverage = ( + None + if wall_delta is None or wall_delta == 0 + else (program_parent.delta.total + equality_parent.delta.total) / wall_delta + ) + coverage_text = ( + "Program + Equality coverage is unavailable because wall time did not change." + if coverage is None + else ( + f"Program + Equality account for {_format_percent(coverage, signed=True)} " + "of this file's wall-time change." + ) + ) + source_count = program_parent.ruleset_count + source_shown = min(source_count, RULESET_CONTRIBUTOR_LIMIT) + source_text = f"Source rules shown: {source_shown}/{source_count}" + source_text += " plus exact Other." if source_count > source_shown else "." + maintenance_count = equality_parent.ruleset_count + maintenance_text = ( + "Maintenance rules shown: none." + if maintenance_count == 0 + else f"Maintenance rules shown: {maintenance_count}/{maintenance_count}." + ) + caption = f"{coverage_text} {source_text} {maintenance_text}" blocks.append( _table( report_id("table", "rulesets", file.sha256, file.fact_directory_sha256), title, - ( - "ruleset", - "baseline", - "candidate", - "delta", - "search_delta", - "apply_delta", - "execution_delta", - "merge_delta", - "rebuild_delta", - ), - ("Ruleset", "Baseline total", "Candidate total", "Total Δ", "S Δ", "A Δ", "Exec Δ", "M Δ", "R Δ"), - tuple( - _row( - report_id("row", "rulesets", file.sha256, file.fact_directory_sha256, row.name), - text_cell(row.name, DEFAULT_RULESET if row.name == "" else row.name), - _duration_estimate_cell(row.baseline), - _duration_estimate_cell(row.candidate), - text_cell(row.delta.total, format_duration(row.delta.total, signed=True)), - text_cell(row.delta.phases.search, format_duration(row.delta.phases.search, signed=True)), - text_cell(row.delta.phases.apply, format_duration(row.delta.phases.apply, signed=True)), - text_cell( - row.delta.phases.unattributed, - format_duration(row.delta.phases.unattributed, signed=True), - ), - text_cell(row.delta.phases.merge, format_duration(row.delta.phases.merge, signed=True)), - text_cell(row.delta.phases.rebuild, format_duration(row.delta.phases.rebuild, signed=True)), - ) - for row in rulesets - ), + ("driver", "delta", "share", "important_phases"), + ("Driver", "Δ", "Wall share", "Important phase changes"), + tuple(_ruleset_report_row(file, row, wall_delta) for row in rulesets), caption=caption, - alignments=("left", "right", "right", "right", "right", "right", "right", "right", "right"), + alignments=("left", "right", "right", "left"), ) ) - return ReportSection("rulesets", "Ruleset comparison", tuple(blocks)) + return ReportSection("rulesets", "Ruleset drivers", tuple(blocks)) + + +def _ruleset_report_row(file: FileSpec, row: RulesetContributorView, wall_delta: float | None) -> ReportRow: + parent = row.kind == "aggregate" + share = None if not parent or wall_delta is None or wall_delta == 0 else row.delta.total / wall_delta + tone = _delta_tone(row.delta.total, share=share) + return _row( + report_id( + "row", + "rulesets", + file.sha256, + file.fact_directory_sha256, + row.kind, + row.mechanism, + row.name, + ), + text_cell( + row.name, + _ruleset_contributor_label(row), + tone="emphasis" if parent else "default", + ), + text_cell(row.delta.total, format_duration(row.delta.total, signed=True), tone=tone), + text_cell(share, _format_percent(share, signed=True) if parent else "", tone=tone), + text_cell(_important_phase_changes(row.delta), tone=tone), + ) + + +def _ruleset_contributor_label(row: RulesetContributorView) -> str: + if row.kind == "aggregate": + return "Program rules — own work" if row.mechanism == "program" else "Equality/rebuild — net" + if row.kind == "native_rebuild": + return "↳ Native rebuild replaced" + if row.kind == "other": + return f"↳ Other ({row.ruleset_count} more source rulesets)" + name = DEFAULT_RULESET if row.name == "" else row.name + return f"↳ {name}" + + +def _important_phase_changes(delta: RulesetDelta) -> str: + labels = ("Assembly", "Search", "Apply", "Execution", "Merge", "Rebuild") + changed = [index for index, value in enumerate(delta.phases) if value != 0] + if not changed: + return "0 ns" + dominant = max(changed, key=lambda index: abs(delta.phases[index])) + threshold = max(1_000_000.0, abs(delta.total) * 0.1) + included = {index for index in changed if abs(delta.phases[index]) >= threshold} + included.add(dominant) + parts = [ + f"{'◆ ' if index == dominant else ''}{labels[index]} {format_duration(delta.phases[index], signed=True)}" + for index in range(len(labels)) + if index in included + ] + if any(index not in included for index in changed): + parts.append("…") + return "; ".join(parts) def report_file_labels(files: Sequence[FileSpec]) -> dict[FileSpec, str]: @@ -468,6 +582,12 @@ def format_duration( return f"{prefix}{_format_scaled(value_ns / divisor, signed=signed)} {unit}" +def _format_delta_ms(value_ns: float | None) -> str: + if value_ns is None: + return NULL + return f"{_format_scaled(value_ns / 1_000_000.0, signed=True)} ms" + + def _format_duration_interval( point_ns: float | None, low_ns: float | None, @@ -530,27 +650,16 @@ def _estimate_cell( return text_cell(point, display) -def _duration_estimate_cell(estimate: Estimate | None) -> ReportCell: - if estimate is None: - return text_cell(None, NULL) - return text_cell(estimate.point, _format_duration_interval(*estimate)) - - -def _phase_estimate_cell( - phase: PhaseEstimate, - *, - attribution: bool, -) -> ReportCell: - duration = _format_duration_interval(*phase.timing, attribution=attribution) - display = duration if phase.wall_share is None else f"{duration} · {_format_percent(phase.wall_share)}" - point = phase.timing.point - tone: CellTone = "warning" if attribution and point is not None and point < 0 else "default" - return text_cell(point, display, tone=tone) - - def _ratio_cell(ratio: RatioEstimate) -> ReportCell: # Retain the point for sorting/filtering while keeping the visible CI cell compact. - return text_cell(ratio.estimate.point, format_ratio_summary(ratio)) + tones: dict[ResultClass, CellTone] = { + "higher": "default", + "invalid": "error", + "lower": "positive", + "point_only": "muted", + "unclear": "muted", + } + return text_cell(ratio.estimate.point, format_ratio_summary(ratio), tone=tones[ratio.result_class]) def format_ratio_summary(ratio: RatioEstimate) -> str: @@ -616,11 +725,11 @@ def _result_cell(result_class: ResultClass, issue: str | None, *, rss: bool) -> else: raise AssertionError(f"unknown result class: {result_class}") tones: dict[ResultClass, CellTone] = { - "higher": "negative", + "higher": "default", "invalid": "error", "lower": "positive", "point_only": "muted", - "unclear": "warning", + "unclear": "muted", } return text_cell(result_class, text, tone=tones[result_class]) diff --git a/benchmarking/reports/render.py b/benchmarking/reports/render.py index 635b174a..f43129f8 100644 --- a/benchmarking/reports/render.py +++ b/benchmarking/reports/render.py @@ -28,7 +28,7 @@ TONE_STYLES: dict[CellTone, str] = { "default": "", "positive": "green", - "negative": "red", + "emphasis": "bold", "warning": "yellow", "error": "bold red", "muted": "dim", @@ -53,7 +53,10 @@ def report_table(title: str | None, *, caption: str | None = None) -> Table: def render_rich_table(table_data: ReportTable, *, show_title: bool = True) -> Table: """Render one catalog table without interpreting its display strings.""" - table = report_table(table_data.title if show_title else None, caption=table_data.caption) + table = report_table( + table_data.title if show_title else None, + caption=table_data.caption, + ) for column in table_data.columns: table.add_column( Text(column.label), @@ -136,8 +139,7 @@ def _markdown_section_parts(section: ReportSection) -> tuple[str, ...]: if section.title is not None: parts.append(f"## {_markdown_heading(section.title)}") for index, block in enumerate(section.blocks): - if index == 0 and _first_table_repeats_section_title(section): - assert isinstance(block, ReportTable) + if isinstance(block, ReportTable) and index == 0 and block.title == section.title: parts.append(render_markdown_table(block, heading_level=None)) else: parts.append(_render_markdown_block(block)) diff --git a/benchmarking/reports/store.py b/benchmarking/reports/store.py index a8012e2a..fca1eda5 100644 --- a/benchmarking/reports/store.py +++ b/benchmarking/reports/store.py @@ -22,29 +22,25 @@ Treatment, ) -type ReportSchemaVersion = Literal[2] -REPORT_SCHEMA_VERSION: Final[ReportSchemaVersion] = 2 +type ReportSchemaVersion = Literal[3] +REPORT_SCHEMA_VERSION: Final[ReportSchemaVersion] = 3 -type TimingSummarySchemaVersion = Literal[2] -TIMING_SUMMARY_SCHEMA_VERSION: Final[TimingSummarySchemaVersion] = 2 +type TimingSummarySchemaVersion = Literal[3] +TIMING_SUMMARY_SCHEMA_VERSION: Final[TimingSummarySchemaVersion] = 3 -class RulesetTimingRecord(TypedDict): - """Persisted engine time for one ruleset.""" +class TimingLeafRecord(TypedDict): + """One exclusive timing leaf with unambiguous path segments.""" - name: str - search_ns: int - apply_ns: int - unattributed_ns: int - merge_ns: int - rebuild_ns: int + path: list[str] + ns: int class TimingSummaryRecord(TypedDict): """Versioned engine timing summary embedded in one successful row.""" schema_version: TimingSummarySchemaVersion - rulesets: list[RulesetTimingRecord] + timings: list[TimingLeafRecord] class ReportRecord(TypedDict): diff --git a/egglog/core-relations/src/free_join/execute.rs b/egglog/core-relations/src/free_join/execute.rs index 83ca6727..f9f352d2 100644 --- a/egglog/core-relations/src/free_join/execute.rs +++ b/egglog/core-relations/src/free_join/execute.rs @@ -454,6 +454,10 @@ impl Database { ..RuleSetReport::default() }; } + // This outer interval includes all per-run execution setup. Search and + // apply are measured inside it, and the serial remainder is reported as + // execution overhead. + let pre_merge_timer = Instant::now(); let match_counter = Arc::new(MatchCounter::new(rule_set.actions.n_ids())); // Trie roots are shared across all plans in this run. Tables are frozen // for the duration, so a given root key always denotes the same subset; @@ -478,7 +482,6 @@ impl Database { (!shared.is_empty()).then(|| Arc::new(TrieCache::with_shared(shared))) }; - let pre_merge_timer = Instant::now(); // let mut rule_reports: HashMap>; let mut rule_reports: HashMap, Vec>; let run_in_parallel = parallelize_db_level_op(self.total_size_estimate); diff --git a/egglog/egglog-bridge/src/lib.rs b/egglog/egglog-bridge/src/lib.rs index 4f3f7eda..7609dd42 100644 --- a/egglog/egglog-bridge/src/lib.rs +++ b/egglog/egglog-bridge/src/lib.rs @@ -853,13 +853,14 @@ impl EGraph { let ts = self.next_ts(); let uf_size_before = self.db.get_table(self.uf_table).len(); - let rule_set_report = + let (assembly_time, rule_set_report) = run_rules_impl(&mut self.db, &mut self.rules, rules, ts, self.report_level)?; if let Some(message) = self.panic_message.lock().unwrap().take() { return Err(PanicError(message).into()); } let mut iteration_report = IterationReport { + assembly_time, rule_set_report, rebuild_time: Duration::ZERO, }; @@ -978,6 +979,7 @@ impl EGraph { ts, ReportLevel::TimeOnly, )? + .1 .changed; } // Reset the rule we did not run. These two should be equivalent. @@ -993,6 +995,7 @@ impl EGraph { ts, ReportLevel::TimeOnly, )? + .1 .changed; for rule in &info.incremental_rebuild_rules { self.rules[*rule].last_run_at = ts; @@ -1066,6 +1069,7 @@ impl EGraph { ts, ReportLevel::TimeOnly, )? + .1 .changed; scratch.clear(); let ts = self.next_ts(); @@ -1082,6 +1086,7 @@ impl EGraph { ts, ReportLevel::TimeOnly, )? + .1 .changed; scratch.clear(); } @@ -2191,7 +2196,8 @@ fn run_rules_impl( rules: &[RuleId], next_ts: Timestamp, report_level: ReportLevel, -) -> Result { +) -> Result<(Duration, RuleSetReport)> { + let assembly_timer = Instant::now(); for rule in rules { let info = &mut rule_info[*rule]; if info.cached_plan.is_none() { @@ -2207,7 +2213,8 @@ fn run_rules_impl( info.last_run_at = next_ts; } let ruleset = rsb.build(); - Ok(db.run_rule_set(&ruleset, report_level)) + let assembly_time = assembly_timer.elapsed(); + Ok((assembly_time, db.run_rule_set(&ruleset, report_level))) } // These markers are just used to make it easy to distinguish time spent in diff --git a/egglog/egglog-reports/src/lib.rs b/egglog/egglog-reports/src/lib.rs index 73b2a7a0..e2b41413 100644 --- a/egglog/egglog-reports/src/lib.rs +++ b/egglog/egglog-reports/src/lib.rs @@ -142,6 +142,9 @@ impl PreMergeTiming { /// Aggregated timing for all iterations of one ruleset. #[derive(Debug, Serialize, Clone, Copy, PartialEq, Eq, Default)] pub struct RulesetTiming { + /// Building the executable ruleset for each invocation, including lazy + /// cached-plan creation on first use. + pub assembly: Duration, /// Execution before staged updates are merged. pub pre_merge: PreMergeTiming, /// Resolving and installing staged updates. @@ -152,10 +155,11 @@ pub struct RulesetTiming { impl RulesetTiming { pub fn total(self) -> Duration { - self.pre_merge.total() + self.merge + self.rebuild + self.assembly + self.pre_merge.total() + self.merge + self.rebuild } fn union(&mut self, other: Self) { + self.assembly += other.assembly; self.pre_merge.union(other.pre_merge); self.merge += other.merge; self.rebuild += other.rebuild; @@ -180,6 +184,8 @@ impl RuleSetReport { #[derive(Debug, Serialize, Clone, Default)] pub struct IterationReport { + /// Preparing this invocation's executable ruleset before execution starts. + pub assembly_time: Duration, pub rule_set_report: RuleSetReport, pub rebuild_time: Duration, } @@ -196,6 +202,14 @@ impl IterationReport { pub fn rules(&self) -> impl Iterator> { self.rule_set_report.rule_reports.keys() } + + /// Total exclusive wall-clock work recorded for this invocation. + pub fn total_time(&self) -> Duration { + self.assembly_time + + self.rule_set_report.pre_merge.total() + + self.rule_set_report.merge_time + + self.rebuild_time + } } /// Running a schedule produces a report of the results. @@ -247,6 +261,7 @@ impl Display for RunReport { } for (ruleset, timing) in &self.ruleset_timings { + let assembly_time = timing.assembly.as_secs_f64(); let merge_time = timing.merge.as_secs_f64(); let rebuild_time = timing.rebuild.as_secs_f64(); match timing.pre_merge { @@ -257,7 +272,7 @@ impl Display for RunReport { } => { writeln!( f, - "Ruleset {ruleset}: search {:.3}s, apply {:.3}s, unattributed {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", + "Ruleset {ruleset}: assembly {assembly_time:.3}s, search {:.3}s, apply {:.3}s, unattributed {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", search.as_secs_f64(), apply.as_secs_f64(), unattributed.as_secs_f64(), @@ -266,7 +281,7 @@ impl Display for RunReport { PreMergeTiming::Combined { elapsed } => { writeln!( f, - "Ruleset {ruleset}: pre-merge {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", + "Ruleset {ruleset}: assembly {assembly_time:.3}s, pre-merge {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", elapsed.as_secs_f64(), )?; } @@ -321,6 +336,7 @@ impl RunReport { report.ruleset_timings.insert( ruleset, RulesetTiming { + assembly: iteration.assembly_time, pre_merge: iteration.rule_set_report.pre_merge, merge: iteration.rule_set_report.merge_time, rebuild: iteration.rebuild_time, @@ -337,6 +353,15 @@ impl RunReport { self.union(RunReport::singleton(ruleset, iteration)); } + /// Total wall-clock work recorded by all ruleset phase timers. + pub fn total_ruleset_time(&self) -> Duration { + self.ruleset_timings + .values() + .copied() + .map(RulesetTiming::total) + .sum() + } + /// Merge two reports. pub fn union(&mut self, other: Self) { self.iterations.extend(other.iterations); @@ -356,29 +381,28 @@ impl RunReport { } } -/// Compact, deterministic timing transport for benchmark runners. +/// One exclusive timing leaf in the benchmark transport. +/// +/// Static mechanism and phase names occupy the first two segments. Ruleset +/// leaves add the exact ruleset name as a third segment. Segments are stored +/// separately so user names containing `/` remain unambiguous. #[derive(Debug, Serialize, Clone, PartialEq, Eq)] -pub struct RulesetTimingV2 { - pub name: String, - pub search_ns: u64, - pub apply_ns: u64, - pub unattributed_ns: u64, - pub merge_ns: u64, - pub rebuild_ns: u64, +pub struct TimingLeafV3 { + pub path: Vec, + pub ns: u64, } -/// Versioned ruleset timing summary for successful egglog runs. +/// Versioned, deterministic timing transport for successful egglog runs. /// -/// V2 includes every name in [`RunReport::ruleset_timings`], preserves the -/// empty name used by the default ruleset, and sorts names lexicographically. -/// Split pre-merge timing must be available for every included ruleset; -/// otherwise construction returns [`PhaseTimingUnavailable`]. Durations are -/// converted to nanoseconds with saturation at [`u64::MAX`], and the ruleset -/// list is never truncated. +/// The values are exclusive wall-clock leaves: their sum can be subtracted +/// once from process wall time to derive residual. Parent totals are never +/// stored. Construction sorts paths lexicographically, rejects duplicate paths +/// as a producer bug, saturates nanoseconds at [`u64::MAX`], and never +/// truncates the leaf list. #[derive(Debug, Serialize, Clone, PartialEq, Eq)] -pub struct TimingSummaryV2 { +pub struct TimingSummaryV3 { pub schema_version: u32, - pub rulesets: Vec, + pub timings: Vec, } /// A requested timing summary contains a ruleset whose split phase timing was @@ -400,39 +424,29 @@ impl Display for PhaseTimingUnavailable { impl std::error::Error for PhaseTimingUnavailable {} -impl TimingSummaryV2 { - pub fn from_run_report(report: &RunReport) -> Result { - let mut timings = report.ruleset_timings.iter().collect::>(); - timings.sort_unstable_by(|(left, _), (right, _)| left.as_ref().cmp(right.as_ref())); - - let rulesets = timings +impl TimingSummaryV3 { + pub fn new(timings: impl IntoIterator, Duration)>) -> Self { + let mut timings = timings .into_iter() - .map(|(name, timing)| { - let PreMergeTiming::Split { - search, - apply, - unattributed, - } = timing.pre_merge - else { - return Err(PhaseTimingUnavailable { - ruleset: name.to_string(), - }); - }; - Ok(RulesetTimingV2 { - search_ns: duration_ns(search), - apply_ns: duration_ns(apply), - unattributed_ns: duration_ns(unattributed), - merge_ns: duration_ns(timing.merge), - rebuild_ns: duration_ns(timing.rebuild), - name: name.to_string(), - }) + .map(|(path, duration)| TimingLeafV3 { + path, + ns: duration_ns(duration), }) - .collect::, PhaseTimingUnavailable>>()?; + .collect::>(); + assert!( + timings.iter().all(|timing| !timing.path.is_empty()), + "timing paths must not be empty" + ); + timings.sort_unstable_by(|left, right| left.path.cmp(&right.path)); + assert!( + timings.windows(2).all(|pair| pair[0].path != pair[1].path), + "duplicate timing path" + ); - Ok(Self { - schema_version: 2, - rulesets, - }) + Self { + schema_version: 3, + timings, + } } } @@ -453,62 +467,53 @@ mod tests { } #[test] - fn timing_summary_v2_exact_json_is_sorted() { - let mut report = RunReport::default(); - report.ruleset_timings.insert( - "zeta".into(), - RulesetTiming { - pre_merge: PreMergeTiming::Split { - search: Duration::new(1, 234), - apply: Duration::ZERO, - unattributed: Duration::from_nanos(89), - }, - ..RulesetTiming::default() - }, - ); - report.ruleset_timings.insert( - "beta".into(), - RulesetTiming { - pre_merge: split(0, 23, 0), - ..RulesetTiming::default() - }, - ); - report.ruleset_timings.insert( - "".into(), - RulesetTiming { - pre_merge: split(0, 0, 0), - merge: Duration::from_nanos(45), - ..RulesetTiming::default() - }, - ); - report.ruleset_timings.insert( - "alpha".into(), - RulesetTiming { - pre_merge: split(0, 0, 0), - rebuild: Duration::from_nanos(67), - ..RulesetTiming::default() - }, - ); - - let summary = TimingSummaryV2::from_run_report(&report).unwrap(); + fn timing_summary_v3_exact_json_is_sorted_and_segmented() { + let summary = TimingSummaryV3::new([ + ( + vec!["program".into(), "search".into(), "rules/λ".into()], + Duration::new(1, 234), + ), + ( + vec!["commands".into(), "check".into()], + Duration::from_nanos(6), + ), + ( + vec!["equality".into(), "rebuild".into(), "rules/λ".into()], + Duration::from_nanos(67), + ), + ( + vec!["frontend".into(), "parse".into()], + Duration::from_nanos(1), + ), + ( + vec!["typecheck".into(), "total".into()], + Duration::from_nanos(2), + ), + ]); let json = serde_json::to_string(&summary).unwrap(); assert_eq!( json, - r#"{"schema_version":2,"rulesets":[{"name":"","search_ns":0,"apply_ns":0,"unattributed_ns":0,"merge_ns":45,"rebuild_ns":0},{"name":"alpha","search_ns":0,"apply_ns":0,"unattributed_ns":0,"merge_ns":0,"rebuild_ns":67},{"name":"beta","search_ns":0,"apply_ns":23,"unattributed_ns":0,"merge_ns":0,"rebuild_ns":0},{"name":"zeta","search_ns":1000000234,"apply_ns":0,"unattributed_ns":89,"merge_ns":0,"rebuild_ns":0}]}"# + r#"{"schema_version":3,"timings":[{"path":["commands","check"],"ns":6},{"path":["equality","rebuild","rules/λ"],"ns":67},{"path":["frontend","parse"],"ns":1},{"path":["program","search","rules/λ"],"ns":1000000234},{"path":["typecheck","total"],"ns":2}]}"# ); } #[test] - fn timing_summary_v2_empty_report_golden() { - let summary = TimingSummaryV2::from_run_report(&RunReport::default()).unwrap(); + fn timing_summary_v3_empty_report_golden() { + let summary = TimingSummaryV3::new([]); let json = serde_json::to_string(&summary).unwrap(); - assert_eq!(json, r#"{"schema_version":2,"rulesets":[]}"#); + assert_eq!(json, r#"{"schema_version":3,"timings":[]}"#); } #[test] - fn timing_summary_v2_aggregates_every_iteration_of_a_ruleset() { + #[should_panic(expected = "timing paths must not be empty")] + fn timing_summary_v3_rejects_an_empty_path() { + TimingSummaryV3::new([(vec![], Duration::ZERO)]); + } + + #[test] + fn run_report_aggregates_every_iteration_of_a_ruleset() { let mut report = RunReport::default(); report.add_iteration( "timed", @@ -519,6 +524,7 @@ mod tests { ..RuleSetReport::default() }, rebuild_time: Duration::from_nanos(17), + assembly_time: Duration::from_nanos(2), }, ); report.add_iteration( @@ -530,92 +536,55 @@ mod tests { ..RuleSetReport::default() }, rebuild_time: Duration::from_nanos(29), + assembly_time: Duration::from_nanos(3), }, ); - let summary = TimingSummaryV2::from_run_report(&report).unwrap(); - assert_eq!( report.ruleset_timings["timed"].pre_merge.total(), Duration::from_nanos(49) ); assert_eq!( report.ruleset_timings["timed"].total(), - Duration::from_nanos(131) - ); - - assert_eq!( - summary.rulesets, - [RulesetTimingV2 { - name: "timed".to_owned(), - search_ns: 30, - apply_ns: 12, - unattributed_ns: 7, - merge_ns: 36, - rebuild_ns: 46, - }] + Duration::from_nanos(136) ); } #[test] - fn timing_summary_v2_does_not_truncate_rulesets() { - let mut report = RunReport::default(); - for index in (0..40).rev() { - report.ruleset_timings.insert( - format!("ruleset-{index:02}").into(), - RulesetTiming { - pre_merge: split(index + 1, 0, 0), - ..RulesetTiming::default() - }, - ); - } - - let summary = TimingSummaryV2::from_run_report(&report).unwrap(); - - assert_eq!(summary.rulesets.len(), 40); - assert_eq!(summary.rulesets.first().unwrap().name, "ruleset-00"); - assert_eq!(summary.rulesets.last().unwrap().name, "ruleset-39"); + fn timing_summary_v3_does_not_truncate_leaves() { + let summary = TimingSummaryV3::new((0..40).rev().map(|index| { + ( + vec![ + "program".into(), + "search".into(), + format!("ruleset-{index:02}"), + ], + Duration::from_nanos(index + 1), + ) + })); + + assert_eq!(summary.timings.len(), 40); + assert_eq!(summary.timings.first().unwrap().path[2], "ruleset-00"); + assert_eq!(summary.timings.last().unwrap().path[2], "ruleset-39"); } #[test] - fn timing_summary_v2_saturates_nanoseconds_to_u64() { - let mut report = RunReport::default(); - report.ruleset_timings.insert( - "long".into(), - RulesetTiming { - pre_merge: PreMergeTiming::Split { - search: Duration::from_secs(u64::MAX), - apply: Duration::ZERO, - unattributed: Duration::ZERO, - }, - ..RulesetTiming::default() - }, - ); + fn timing_summary_v3_saturates_nanoseconds_to_u64() { + let summary = TimingSummaryV3::new([( + vec!["program".into(), "search".into(), "long".into()], + Duration::from_secs(u64::MAX), + )]); - let summary = TimingSummaryV2::from_run_report(&report).unwrap(); - - assert_eq!(summary.rulesets[0].search_ns, u64::MAX); + assert_eq!(summary.timings[0].ns, u64::MAX); } #[test] - fn timing_summary_v2_rejects_unavailable_split_timing() { - let mut report = RunReport::default(); - report.ruleset_timings.insert( - "default".into(), - RulesetTiming { - pre_merge: PreMergeTiming::Combined { - elapsed: Duration::from_nanos(42), - }, - ..RulesetTiming::default() - }, - ); - - assert_eq!( - TimingSummaryV2::from_run_report(&report), - Err(PhaseTimingUnavailable { - ruleset: "default".to_owned(), - }) - ); + #[should_panic(expected = "duplicate timing path")] + fn timing_summary_v3_rejects_duplicate_paths() { + TimingSummaryV3::new([ + (vec!["commands".into(), "check".into()], Duration::ZERO), + (vec!["commands".into(), "check".into()], Duration::ZERO), + ]); } #[test] @@ -630,6 +599,7 @@ mod tests { ..RuleSetReport::default() }, rebuild_time: Duration::from_nanos(11), + assembly_time: Duration::from_nanos(2), }, ); report.add_iteration( @@ -643,12 +613,14 @@ mod tests { ..RuleSetReport::default() }, rebuild_time: Duration::from_nanos(17), + assembly_time: Duration::from_nanos(3), }, ); assert_eq!( report.ruleset_timings["mixed"], RulesetTiming { + assembly: Duration::from_nanos(5), pre_merge: PreMergeTiming::Combined { elapsed: Duration::from_nanos(11), }, @@ -658,7 +630,7 @@ mod tests { ); assert_eq!( report.ruleset_timings["mixed"].total(), - Duration::from_nanos(59) + Duration::from_nanos(64) ); } } diff --git a/egglog/src/cli.rs b/egglog/src/cli.rs index 55edefc4..50388221 100644 --- a/egglog/src/cli.rs +++ b/egglog/src/cli.rs @@ -3,7 +3,6 @@ use std::io::{self, BufRead, BufReader, IsTerminal, Read, Write}; use std::str::FromStr; use clap::Parser; -use egglog_reports::TimingSummaryV2; use env_logger::Env; use std::path::PathBuf; @@ -234,11 +233,10 @@ pub fn cli(mut egraph: EGraph) { } if let Some(summary_path) = args.timing_summary { - let summary = TimingSummaryV2::from_run_report(egraph.get_overall_run_report()) - .unwrap_or_else(|error| { - log::error!("failed to create timing summary: {error}"); - std::process::exit(1); - }); + let summary = egraph.timing_summary().unwrap_or_else(|error| { + log::error!("failed to create timing summary: {error}"); + std::process::exit(1); + }); let mut file = std::fs::File::create(&summary_path) .unwrap_or_else(|_| panic!("Failed to create timing summary file at {summary_path:?}")); serde_json::to_writer(&mut file, &summary).expect("Failed to serialize timing summary"); diff --git a/egglog/src/lib.rs b/egglog/src/lib.rs index 25aae034..c5b31226 100644 --- a/egglog/src/lib.rs +++ b/egglog/src/lib.rs @@ -8,6 +8,7 @@ pub mod constraint; mod core; mod exec_state; pub mod extract; +mod phase_timers; pub mod prelude; mod proofs; @@ -42,7 +43,9 @@ use egglog_ast::util::ListDisplay; use egglog_bridge::{ColumnTy, QueryEntry}; use egglog_core_relations as core_relations; use egglog_numeric_id as numeric_id; -use egglog_reports::{ReportLevel, RunReport}; +use egglog_reports::{ + PhaseTimingUnavailable, PreMergeTiming, ReportLevel, RunReport, TimingSummaryV3, +}; pub use exec_state::{ Context, Core, Enode, FullState, FunctionEntry, PureState, Read, ReadState, Write, WriteState, }; @@ -71,6 +74,7 @@ use std::iter::once; use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; +use std::time::Instant; pub use termdag::{OrdTerm, Term, TermDag, TermId}; use thiserror::Error; use typechecking::FuncType; @@ -318,12 +322,19 @@ pub struct EGraph { pushed_egraph: Option>, functions: IndexMap, rulesets: IndexMap, + /// The semantic responsibility of each declared ruleset's execution. + ruleset_timing_roles: IndexMap, pub fact_directory: Option, pub seminaive: bool, pub no_decomp: bool, type_info: TypeInfo, /// The run report unioned over all runs so far. overall_run_report: RunReport, + /// Roles for every ruleset present in `overall_run_report`, including work + /// performed inside a scope that has since been popped. + overall_ruleset_timing_roles: IndexMap, + /// Exclusive process work outside ruleset execution. + phase_timings: phase_timers::PhaseTimings, schedulers: DenseIdMap, commands: IndexMap>, extension_state: HashMap>, @@ -439,10 +450,13 @@ impl EGraph { pushed_egraph: Default::default(), functions: Default::default(), rulesets: Default::default(), + ruleset_timing_roles: Default::default(), fact_directory: None, seminaive: true, no_decomp: false, overall_run_report: Default::default(), + overall_ruleset_timing_roles: Default::default(), + phase_timings: Default::default(), type_info: Default::default(), schedulers: Default::default(), commands: Default::default(), @@ -564,6 +578,8 @@ impl EGraph { eg.rulesets .insert("".into(), Ruleset::Rules(Default::default())); + eg.ruleset_timing_roles + .insert("".into(), phase_timers::RulesetTimingRole::Program); // The generic `get-fresh!` mint primitive is registered on every e-graph. // Doing it here — rather than per-eq-sort — means it is present whenever @@ -841,6 +857,12 @@ impl EGraph { Some(mut e) => { // Preserve the overall report from the popped egraph std::mem::swap(&mut self.overall_run_report, &mut e.overall_run_report); + std::mem::swap( + &mut self.overall_ruleset_timing_roles, + &mut e.overall_ruleset_timing_roles, + ); + // Work performed in the popped scope still belongs to this run. + std::mem::swap(&mut self.phase_timings, &mut e.phase_timings); // Preserve the symbol generator so that fresh symbols // generated after pop don't collide with ones generated before pop. std::mem::swap(&mut self.parser.symbol_gen, &mut e.parser.symbol_gen); @@ -1387,10 +1409,25 @@ impl EGraph { .map_err(|e| Error::BackendError(e.to_string()))?; let report = RunReport::singleton(ruleset, iteration_report); + self.record_ruleset_timing_role(ruleset); self.overall_run_report.union(report.clone()); Ok(report) } + fn record_ruleset_timing_role(&mut self, ruleset: &str) { + let role = self.ruleset_timing_roles[ruleset]; + match self.overall_ruleset_timing_roles.entry(ruleset.to_owned()) { + Entry::Occupied(entry) => assert_eq!( + *entry.get(), + role, + "a ruleset's timing role changed after it was recorded" + ), + Entry::Vacant(entry) => { + entry.insert(role); + } + } + } + fn add_rule(&mut self, rule: ast::ResolvedRule) -> Result { // The `:naive` rule option opts a single rule out of seminaive // evaluation. This widens primitive-context selection from @@ -1869,18 +1906,55 @@ impl EGraph { Ok(result) } - fn add_combined_ruleset(&mut self, name: String, rulesets: Vec) { + fn add_combined_ruleset( + &mut self, + span: &Span, + name: String, + rulesets: Vec, + ) -> Result<(), Error> { + let mut timing_role = None; + for ruleset in &rulesets { + let role = self + .ruleset_timing_roles + .get(ruleset) + .copied() + .ok_or_else(|| Error::NoSuchRuleset(ruleset.clone(), span.clone()))?; + if timing_role.is_some_and(|expected| expected != role) { + return Err(Error::MixedRulesetResponsibilities(name, span.clone())); + } + timing_role = Some(role); + } match self.rulesets.entry(name.clone()) { Entry::Occupied(_) => panic!("Ruleset '{name}' was already present"), Entry::Vacant(e) => e.insert(Ruleset::Combined(rulesets)), }; + self.ruleset_timing_roles.insert( + name, + timing_role.unwrap_or(phase_timers::RulesetTimingRole::Program), + ); + Ok(()) } fn add_ruleset(&mut self, name: String) { + let proof_names = &self.proof_state.proof_names; + let timing_role = if [ + &proof_names.path_compress_ruleset_name, + &proof_names.rebuilding_ruleset_name, + &proof_names.rebuilding_cleanup_ruleset_name, + &proof_names.subsume_ruleset_name, + ] + .iter() + .any(|generated| generated.as_str() == name) + { + phase_timers::RulesetTimingRole::EqualityMaintenance + } else { + phase_timers::RulesetTimingRole::Program + }; match self.rulesets.entry(name.clone()) { Entry::Occupied(_) => panic!("Ruleset '{name}' was already present"), Entry::Vacant(e) => e.insert(Ruleset::Rules(Default::default())), }; + self.ruleset_timing_roles.insert(name, timing_role); } fn check_facts(&mut self, span: &Span, facts: &[ResolvedFact]) -> Result<(), Error> { @@ -1930,7 +2004,9 @@ impl EGraph { let run_result = self.backend.run_rules(&[id]); self.backend.free_rule(id); self.backend.free_external_func(ext_id); - run_result.map_err(|e| Error::BackendError(e.to_string()))?; + let iteration_report = run_result.map_err(|e| Error::BackendError(e.to_string()))?; + self.phase_timings + .add(phase_timers::COMMANDS_CHECK, iteration_report.total_time()); let ext_sc_val = ext_sc.lock().unwrap().take(); let matched = matches!(ext_sc_val, Some(())); @@ -1946,6 +2022,59 @@ impl EGraph { } fn run_command(&mut self, command: ResolvedNCommand) -> Result, Error> { + enum CommandPhase { + Install, + Actions, + Check, + Other, + } + + let phase = match &command { + ResolvedNCommand::Sort { .. } + | ResolvedNCommand::Function(_) + | ResolvedNCommand::Index { .. } + | ResolvedNCommand::AddRuleset(..) + | ResolvedNCommand::UnstableCombinedRuleset(..) + | ResolvedNCommand::NormRule { .. } => CommandPhase::Install, + ResolvedNCommand::CoreAction(_) + | ResolvedNCommand::CoreActions(_) + | ResolvedNCommand::Input { .. } => CommandPhase::Actions, + ResolvedNCommand::Check(..) => CommandPhase::Check, + _ => CommandPhase::Other, + }; + let command_timer = Instant::now(); + let process_before = self.phase_timings.total(); + let ruleset_before = self.overall_run_report.total_ruleset_time(); + let result = self.run_command_inner(command); + let nested_process = self.phase_timings.total().saturating_sub(process_before); + let nested_rulesets = self + .overall_run_report + .total_ruleset_time() + .saturating_sub(ruleset_before); + let own_time = command_timer + .elapsed() + .saturating_sub(nested_process + nested_rulesets); + match phase { + CommandPhase::Install => self + .phase_timings + .add(phase_timers::FRONTEND_INSTALL, own_time), + CommandPhase::Actions => self + .phase_timings + .add(phase_timers::COMMANDS_ACTIONS, own_time), + CommandPhase::Check => self + .phase_timings + .add(phase_timers::COMMANDS_CHECK, own_time), + CommandPhase::Other => self + .phase_timings + .add(phase_timers::COMMANDS_OTHER, own_time), + } + result + } + + fn run_command_inner( + &mut self, + command: ResolvedNCommand, + ) -> Result, Error> { match command { // Sorts are already declared during typechecking ResolvedNCommand::Sort { @@ -1992,8 +2121,8 @@ impl EGraph { self.add_ruleset(name.clone()); log::info!("Declared ruleset {name}."); } - ResolvedNCommand::UnstableCombinedRuleset(_span, name, others) => { - self.add_combined_ruleset(name.clone(), others); + ResolvedNCommand::UnstableCombinedRuleset(span, name, others) => { + self.add_combined_ruleset(&span, name.clone(), others)?; log::info!("Declared ruleset {name}."); } ResolvedNCommand::NormRule { rule } => { @@ -2515,7 +2644,10 @@ impl EGraph { if let Some(original_typechecking) = self.proof_state.original_typechecking.as_mut() { // Typecheck using the original egraph // TODO this is ugly- we don't need an entire e-graph just for type information. + let typecheck_timer = Instant::now(); let typechecked = original_typechecking.typecheck_program(&desugared)?; + self.phase_timings + .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); for command in &typechecked { if let Err(reason) = command_supports_proof_encoding( @@ -2532,7 +2664,10 @@ impl EGraph { Ok(proof_form(typechecked, &mut self.parser.symbol_gen)) } else { + let typecheck_timer = Instant::now(); let mut typechecked = self.typecheck_program(&desugared)?; + self.phase_timings + .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); typechecked = remove_globals::remove_globals(typechecked, &mut self.parser.symbol_gen); for command in &typechecked { @@ -2546,6 +2681,18 @@ impl EGraph { /// Leverages previous type information in the [`EGraph`] to do so, adding new type information. /// When will_run is true, adds to `desugared_commands_run_so_far`, which is used for proof checking. fn resolve_command(&mut self, command: Command) -> Result { + let lowering_timer = Instant::now(); + let nested_before = self.phase_timings.total(); + let resolved = self.resolve_command_inner(command); + let nested = self.phase_timings.total().saturating_sub(nested_before); + self.phase_timings.add( + phase_timers::FRONTEND_OTHER, + lowering_timer.elapsed().saturating_sub(nested), + ); + resolved + } + + fn resolve_command_inner(&mut self, command: Command) -> Result { let resolved_before_proofs = self.resolve_command_before_proofs(command)?; // Add term encoding when it is enabled @@ -2593,7 +2740,10 @@ impl EGraph { } // Now typecheck using self, adding term type information. + let typecheck_timer = Instant::now(); let desugared_typechecked = self.typecheck_program(&desugared)?; + self.phase_timings + .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); // Remove the globals the term encoding itself introduced (its minted // `let`s), the same way source-level globals were removed above. let desugared_typechecked = remove_globals::remove_globals( @@ -2630,20 +2780,26 @@ impl EGraph { .as_ref() .map(|egraph| &egraph.type_info) .unwrap_or(&self.type_info); + let macro_timer = Instant::now(); let macro_expanded = self.command_macros.apply( before_expanded_command, &mut self.parser.symbol_gen, macro_type_info, - )?; + ); + self.phase_timings + .add(phase_timers::FRONTEND_OTHER, macro_timer.elapsed()); + let macro_expanded = macro_expanded?; for command in macro_expanded { // handle include specially- we keep them as-is for desugaring if let Command::Include(span, file) = &command { + let include_timer = Instant::now(); let s = std::fs::read_to_string(file) - .map_err(|e| Error::IoError(file.clone().into(), e, span.clone()))?; - let included_program = self - .parser - .get_program_from_string(Some(file.clone()), &s)?; + .map_err(|e| Error::IoError(file.clone().into(), e, span.clone())); + self.phase_timings + .add(phase_timers::FRONTEND_OTHER, include_timer.elapsed()); + let s = s?; + let included_program = self.parse_program_timed(Some(file.clone()), &s)?; // run program internal on these include commands let resolved = self.process_program_internal(included_program, run_commands)?; outputs.extend(resolved.outputs); @@ -2697,7 +2853,7 @@ impl EGraph { filename: Option, input: &str, ) -> Result, Error> { - let parsed = self.parser.get_program_from_string(filename, input)?; + let parsed = self.parse_program_timed(filename, input)?; let res = self.process_program_internal(parsed, false)?; Ok(res.resolved.into_iter().map(|c| c.to_command()).collect()) } @@ -2708,8 +2864,7 @@ impl EGraph { filename: Option, input: &str, ) -> Result, Error> { - let parsed = self.parser.get_program_from_string(filename, input)?; - Ok(parsed) + self.parse_program_timed(filename, input) } /// Takes a source program `input`, parses it, runs it, and returns a list of messages. @@ -2722,10 +2877,24 @@ impl EGraph { filename: Option, input: &str, ) -> Result, Error> { - let parsed = self.parser.get_program_from_string(filename, input)?; + let parsed = self.parse_program_timed(filename, input)?; self.run_program(parsed) } + /// Parse through the single accounting boundary shared by source, include, + /// and generated term-encoding text. + pub(crate) fn parse_program_timed( + &mut self, + filename: Option, + input: &str, + ) -> Result, Error> { + let parse_timer = Instant::now(); + let parsed = self.parser.get_program_from_string(filename, input); + self.phase_timings + .add(phase_timers::FRONTEND_PARSE, parse_timer.elapsed()); + Ok(parsed?) + } + /// Get the number of tuples in the database. /// pub fn num_tuples(&self) -> usize { @@ -2780,6 +2949,55 @@ impl EGraph { &self.overall_run_report } + pub(crate) fn timing_summary(&self) -> Result { + let mut leaves = self.phase_timings.timing_leaves(); + for (ruleset, timing) in &self.overall_run_report.ruleset_timings { + let PreMergeTiming::Split { + search, + apply, + unattributed, + } = timing.pre_merge + else { + return Err(PhaseTimingUnavailable { + ruleset: ruleset.to_string(), + }); + }; + let role = self + .overall_ruleset_timing_roles + .get(ruleset.as_ref()) + .unwrap_or_else(|| panic!("missing timing role for ruleset {ruleset:?}")); + let responsibility = match role { + phase_timers::RulesetTimingRole::Program => "program", + phase_timers::RulesetTimingRole::EqualityMaintenance => "equality", + }; + for (phase, duration) in [ + ("assembly", timing.assembly), + ("search", search), + ("apply", apply), + ("execution", unattributed), + ("merge", timing.merge), + ] { + leaves.push(( + vec![ + responsibility.to_owned(), + phase.to_owned(), + ruleset.to_string(), + ], + duration, + )); + } + leaves.push(( + vec![ + "equality".to_owned(), + "rebuild".to_owned(), + ruleset.to_string(), + ], + timing.rebuild, + )); + } + Ok(TimingSummaryV3::new(leaves)) + } + /// Convert from an egglog value to a Rust type. /// This method assumes `x` belongs to sort `T`. pub fn value_to_base(&self, x: Value) -> T { @@ -2987,6 +3205,7 @@ impl EGraph { self.backend.free_rule(rule.1); } } + self.ruleset_timing_roles.swap_remove(&ruleset); outcome?; let Some(mutex) = Arc::into_inner(results) else { @@ -3559,6 +3778,8 @@ pub enum Error { CheckError(Vec, Span), #[error("{1}\nNo such ruleset: {0}")] NoSuchRuleset(String, Span), + #[error("{1}\nCombined ruleset {0} mixes program and equality-maintenance rulesets")] + MixedRulesetResponsibilities(String, Span), #[error( "{1}\nAttempted to add a rule to combined ruleset {0}. Combined rulesets may only depend on other rulesets." )] @@ -3621,6 +3842,29 @@ mod tests { use crate::PureState; + #[test] + fn encoded_source_typecheck_is_charged_to_the_outer_egraph() { + let mut egraph = EGraph::new_with_term_encoding(); + + egraph + .parse_and_run_program(None, "(datatype Math (Num i64)) (let value (Num 1))") + .unwrap(); + + assert!( + egraph.phase_timings.leaves[phase_timers::FRONTEND_PARSE] > std::time::Duration::ZERO + ); + assert!(egraph.phase_timings.leaves[phase_timers::TYPECHECK] > std::time::Duration::ZERO); + assert!( + egraph.phase_timings.leaves[phase_timers::FRONTEND_OTHER] > std::time::Duration::ZERO + ); + let source_checker = egraph.proof_state.original_typechecking.as_ref().unwrap(); + assert_eq!( + source_checker.phase_timings.leaves[phase_timers::TYPECHECK], + std::time::Duration::ZERO, + "the child checker must not retain time omitted from the outer summary" + ); + } + #[derive(Clone)] struct InnerProduct { vec: ArcSort, @@ -4120,6 +4364,36 @@ mod tests { assert!(matches!(err, Error::NoSuchRuleset(name, _) if name == "test2")); } + #[test] + fn test_combined_ruleset_with_undefined_member_errors() { + let err = EGraph::default() + .parse_and_run_program(None, "(unstable-combined-ruleset combined missing)") + .unwrap_err(); + assert!(matches!(err, Error::NoSuchRuleset(name, _) if name == "missing")); + } + + #[test] + fn test_combined_ruleset_with_mixed_responsibilities_errors() { + let mut egraph = EGraph::default(); + let maintenance = egraph + .proof_state + .proof_names + .rebuilding_ruleset_name + .clone(); + egraph.add_ruleset("program".into()); + egraph.add_ruleset(maintenance.clone()); + + let err = egraph + .add_combined_ruleset( + &span!(), + "mixed".into(), + vec!["program".into(), maintenance], + ) + .unwrap_err(); + + assert!(matches!(err, Error::MixedRulesetResponsibilities(name, _) if name == "mixed")); + } + #[test] fn test_duplicate_rule_name_errors() { let err = EGraph::default() diff --git a/egglog/src/phase_timers.rs b/egglog/src/phase_timers.rs new file mode 100644 index 00000000..b1d43764 --- /dev/null +++ b/egglog/src/phase_timers.rs @@ -0,0 +1,74 @@ +//! Exclusive wall-clock accounting outside ruleset execution. +//! +//! Static slices make paths the recording structure without allocating at +//! timer sites. The persisted summary converts them to owned path segments. + +use std::time::Duration; + +use indexmap::IndexMap; + +pub(crate) type TimingPath = &'static [&'static str]; + +pub(crate) const TYPECHECK: TimingPath = &["typecheck", "total"]; +pub(crate) const FRONTEND_PARSE: TimingPath = &["frontend", "parse"]; +pub(crate) const FRONTEND_OTHER: TimingPath = &["frontend", "other"]; +pub(crate) const FRONTEND_INSTALL: TimingPath = &["frontend", "install"]; +pub(crate) const COMMANDS_ACTIONS: TimingPath = &["commands", "actions"]; +pub(crate) const COMMANDS_CHECK: TimingPath = &["commands", "check"]; +pub(crate) const COMMANDS_OTHER: TimingPath = &["commands", "other"]; + +const STABLE_PROCESS_PATHS: [TimingPath; 7] = [ + TYPECHECK, + FRONTEND_PARSE, + FRONTEND_OTHER, + FRONTEND_INSTALL, + COMMANDS_ACTIONS, + COMMANDS_CHECK, + COMMANDS_OTHER, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RulesetTimingRole { + Program, + EqualityMaintenance, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PhaseTimings { + pub(crate) leaves: IndexMap, +} + +impl Default for PhaseTimings { + fn default() -> Self { + Self { + leaves: STABLE_PROCESS_PATHS + .into_iter() + .map(|path| (path, Duration::ZERO)) + .collect(), + } + } +} + +impl PhaseTimings { + /// Accumulate one exclusive interval under exactly one path. + pub(crate) fn add(&mut self, path: TimingPath, duration: Duration) { + assert!(!path.is_empty(), "timing paths must not be empty"); + *self.leaves.entry(path).or_default() += duration; + } + + pub(crate) fn total(&self) -> Duration { + self.leaves.values().copied().sum() + } + + pub(crate) fn timing_leaves(&self) -> Vec<(Vec, Duration)> { + self.leaves + .iter() + .map(|(path, duration)| { + ( + path.iter().map(|segment| (*segment).to_owned()).collect(), + *duration, + ) + }) + .collect() + } +} diff --git a/egglog/src/proofs/proof_encoding_helpers.rs b/egglog/src/proofs/proof_encoding_helpers.rs index 35fb7523..aa7a80ab 100644 --- a/egglog/src/proofs/proof_encoding_helpers.rs +++ b/egglog/src/proofs/proof_encoding_helpers.rs @@ -472,7 +472,7 @@ impl ProofInstrumentor<'_> { pub(crate) fn parse_program(&mut self, input: &str) -> Vec { self.egraph.parser.ensure_no_reserved_symbols = false; - let res = self.egraph.parser.get_program_from_string(None, input); + let res = self.egraph.parse_program_timed(None, input); self.egraph.parser.ensure_no_reserved_symbols = true; // This program is generated internally by term encoding, so a parse diff --git a/egglog/src/scheduler.rs b/egglog/src/scheduler.rs index f5d1c608..03283506 100644 --- a/egglog/src/scheduler.rs +++ b/egglog/src/scheduler.rs @@ -309,6 +309,7 @@ impl EGraph { self.schedulers = schedulers; if let Ok(report) = &result { + self.record_ruleset_timing_role(ruleset); self.overall_run_report.union(report.clone()); } result diff --git a/egglog/tests/integration_test.rs b/egglog/tests/integration_test.rs index fa234d7b..bf3eb661 100644 --- a/egglog/tests/integration_test.rs +++ b/egglog/tests/integration_test.rs @@ -917,7 +917,7 @@ fn test_print_stats() { let outputs = EGraph::default().parse_and_run_program(None, s).unwrap(); assert_eq!( outputs[1].to_string(), - "Overall statistics:\nRuleset : search 0.000s, apply 0.000s, unattributed 0.000s, merge 0.000s, rebuild 0.000s\n" + "Overall statistics:\nRuleset : assembly 0.000s, search 0.000s, apply 0.000s, unattributed 0.000s, merge 0.000s, rebuild 0.000s\n" ); } diff --git a/egglog/tests/timing_summary_cli.rs b/egglog/tests/timing_summary_cli.rs index 4133cb20..8b27d935 100644 --- a/egglog/tests/timing_summary_cli.rs +++ b/egglog/tests/timing_summary_cli.rs @@ -30,6 +30,121 @@ fn assert_duration(value: &serde_json::Value) { assert!(duration["nanos"].is_u64()); } +fn timing_leaf(summary: &serde_json::Value, path: &[&str]) -> u64 { + summary["timings"] + .as_array() + .unwrap() + .iter() + .find(|leaf| { + leaf["path"] + .as_array() + .unwrap() + .iter() + .map(|segment| segment.as_str().unwrap()) + .eq(path.iter().copied()) + }) + .unwrap_or_else(|| panic!("missing timing leaf {path:?}"))["ns"] + .as_u64() + .unwrap() +} + +#[test] +fn checks_have_the_same_command_timing_path_with_and_without_term_encoding() { + let program = r#" + (relation item (i64)) + (item 1) + (check (item 1)) + "#; + + for (label, treatment_flags) in [("off", &[][..]), ("term", &["--term-encoding"][..])] { + let directory = temporary_directory(label); + let program_path = directory.join("program.egg"); + let summary_path = directory.join("summary.json"); + std::fs::write(&program_path, program).unwrap(); + let mut arguments = treatment_flags.iter().map(Path::new).collect::>(); + arguments.extend([ + Path::new("--timing-summary"), + summary_path.as_path(), + program_path.as_path(), + ]); + + let output = run_egglog(&arguments); + assert!( + output.status.success(), + "egglog failed in {label} mode: {}", + String::from_utf8_lossy(&output.stderr) + ); + let summary: serde_json::Value = + serde_json::from_slice(&std::fs::read(&summary_path).unwrap()).unwrap(); + + assert!(timing_leaf(&summary, &["commands", "check"]) > 0); + assert!(!summary["timings"].as_array().unwrap().iter().any(|leaf| { + let path = leaf["path"].as_array().unwrap(); + path.first().and_then(serde_json::Value::as_str) == Some("program") + && path + .last() + .and_then(serde_json::Value::as_str) + .is_some_and(|name| name.contains("check_facts_ruleset")) + })); + + std::fs::remove_dir_all(directory).unwrap(); + } +} + +#[test] +fn encoded_equality_rulesets_are_tagged_by_role_not_mixed_with_program_rules() { + let directory = temporary_directory("equality-role"); + let program_path = directory.join("program.egg"); + let summary_path = directory.join("summary.json"); + std::fs::write( + &program_path, + r#" + (datatype Math (Num i64)) + (let one (Num 1)) + (let two (Num 2)) + (union one two) + (run 1) + "#, + ) + .unwrap(); + + let output = run_egglog(&[ + Path::new("--term-encoding"), + Path::new("--timing-summary"), + &summary_path, + &program_path, + ]); + assert!( + output.status.success(), + "term-encoded egglog failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let summary: serde_json::Value = + serde_json::from_slice(&std::fs::read(&summary_path).unwrap()).unwrap(); + let maintenance_names = summary["timings"] + .as_array() + .unwrap() + .iter() + .filter_map(|leaf| { + let path = leaf["path"].as_array().unwrap(); + (path.len() == 3 + && path[0].as_str() == Some("equality") + && path[1].as_str() != Some("rebuild")) + .then(|| path[2].as_str().unwrap().to_owned()) + }) + .collect::>(); + + assert!(!maintenance_names.is_empty()); + assert!(!summary["timings"].as_array().unwrap().iter().any(|leaf| { + let path = leaf["path"].as_array().unwrap(); + path.len() == 3 + && path[0].as_str() == Some("program") + && maintenance_names.contains(path[2].as_str().unwrap()) + })); + + std::fs::remove_dir_all(directory).unwrap(); +} + #[test] fn timing_summary_is_compact_and_works_with_every_report_level() { let program = r#" @@ -72,22 +187,37 @@ fn timing_summary_is_compact_and_works_with_every_report_level() { let summary: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(summary.as_object().unwrap().len(), 2); - assert_eq!(summary["schema_version"], 2); - let rulesets = summary["rulesets"].as_array().unwrap(); - assert_eq!( - rulesets - .iter() - .map(|ruleset| ruleset["name"].as_str().unwrap()) - .collect::>(), - ["alpha", "zeta"] - ); - for ruleset in rulesets { - assert_eq!(ruleset.as_object().unwrap().len(), 6); - assert!(ruleset["search_ns"].is_u64()); - assert!(ruleset["apply_ns"].is_u64()); - assert!(ruleset["unattributed_ns"].is_u64()); - assert!(ruleset["merge_ns"].is_u64()); - assert!(ruleset["rebuild_ns"].is_u64()); + assert_eq!(summary["schema_version"], 3); + let timings = summary["timings"].as_array().unwrap(); + assert_eq!(timings.len(), 19); + let paths = timings + .iter() + .map(|leaf| { + leaf["path"] + .as_array() + .unwrap() + .iter() + .map(|segment| segment.as_str().unwrap().to_owned()) + .collect::>() + }) + .collect::>(); + assert!(paths.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(timing_leaf(&summary, &["commands", "check"]), 0); + for path in [ + &["frontend", "parse"][..], + &["frontend", "other"], + &["frontend", "install"], + &["typecheck", "total"], + &["commands", "actions"], + &["commands", "other"], + ] { + assert!(timing_leaf(&summary, path) > 0, "expected nonzero {path:?}"); + } + for ruleset in ["alpha", "zeta"] { + for phase in ["assembly", "search", "apply", "execution", "merge"] { + timing_leaf(&summary, &["program", phase, ruleset]); + } + timing_leaf(&summary, &["equality", "rebuild", ruleset]); } let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&report_path).unwrap()).unwrap(); @@ -196,8 +326,8 @@ fn stdin_program_writes_timing_summary() { let bytes = std::fs::read(&summary_path).unwrap(); assert_eq!(bytes.last(), Some(&b'\n')); let summary: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(summary["schema_version"], 2); - assert!(summary["rulesets"].is_array()); + assert_eq!(summary["schema_version"], 3); + assert!(summary["timings"].is_array()); std::fs::remove_dir_all(directory).unwrap(); } diff --git a/encoding-architecture-bridge.md b/encoding-architecture-bridge.md new file mode 100644 index 00000000..7eb425de --- /dev/null +++ b/encoding-architecture-bridge.md @@ -0,0 +1,577 @@ +# Encodings as semantics, fusion as engineering + +- Status: draft architecture note for paper and maintainer review +- Date: 2026-08-12 +- Audience: egglog maintainers and the encoding-paper authors +- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` +- Implementation permission: none; this document proposes experiments and gates +- Companion: [`term-encoding-unification.md`](term-encoding-unification.md) +- Incremental PR sequence: + [`incremental-unification-pr-roadmap.md`](incremental-unification-pr-roadmap.md) +- Current overhead decomposition: + [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) + +## Executive decision + +Do not choose between the paper's encoding story and a fast, single-path +implementation. Separate two things that the current implementation conflates: + +1. An **encoding is a semantic compiler pass**. It translates a program in an + extended language to a smaller logical language, carries an origin map, and + has a correctness argument. Proofs and slotted e-graphs remain encodings in + this sense and compose in a specified order. +2. A **literal encoded program is only one physical implementation** of that + pass. Production egglog should stage and fuse the encoded operations into + its native tables, union-find, rebuild indexes, and compact evidence arenas. + It should not materialize every administrative relation and then execute the + generated maintenance rules as ordinary user rules. + +The proposed end state is therefore: + +```text +surface program + | + | E_slot (when slotted semantics are requested) + v +typed core program + slot interpretation + | + | E_proof (when proof production is requested) + v +typed encoded program + origin map + optional proof skeletons + | + | normalize, erase unused decorations, and fuse + v +one native physical plan + | + v +one execution engine +``` + +For the paper, the typed encoded program can be printed as ordinary core +egglog and run as a reference semantics. For production, the same artifact is +lowered to a fused plan. The printer is not a separately maintained executor or +a second frontend. + +This permits an honest use of the word *encoding*: the logical translation is +always the source of semantics, while compiler correctness justifies executing +an optimized implementation of it. Compilers do not stop being compilers when +they fuse intermediate allocations away. + +## What should be deleted, and what should survive + +The goal is not to delete the idea of term, proof, or slotted encoding. It is to +delete independent implementations of the same language semantics. + +### Delete from the production path + +- the cloned typechecking `EGraph` and second desugar/typecheck pipeline; +- source-string or source-AST expansion as a prerequisite for execution; +- term-only and proof modes as separately executed egglog programs; +- proof-disabled `Unit` columns threaded through all generated tables; +- generated term/view/`@UF` tables when the native table and union-find already + store the same logical information; +- generated occurrence indexes, path-compression rules, rebuild rules, cleanup + rules, subsumption rules, and schedule injection; +- proof nodes stored as ordinary database rows when a compact side arena or + rule-firing receipt is sufficient; +- `proof_check_program` as a second full command stream; +- backend-selection and lowest-common-denominator abstractions if the project + is in fact removing the alternate backends; +- generic always-on provenance or causal-slicing infrastructure as the + foundation for proofs. + +### Keep as semantic and validation assets + +- the pure equality, proof, and slotted translations; +- the equations and invariants currently documented by the term encoding; +- a deterministic printer for the typed encoded IR, usable by paper examples, + differential tests, and debugging; +- the proof algebra, simplifier, extractor, and independent checker; +- proof skeletons and stable source-rule identities; +- slotted renamings, symmetry/group checks, and an explicit account of fresh + slots; +- origin and interpretation metadata that maps compiled rules and + substitutions back to the source language; +- literal-vs-fused parity tests. + +The reference printer may live in the repository, a paper-artifact crate, or a +test-only feature. That packaging choice is secondary. It must be derived from +the same typed pass result, rather than becoming a second implementation that +can drift. + +## What "one execution path" means + +One path does not mean one feature-free IR node or one enormous interpreter. +It means: + +- one parse, resolution, normalization, and typechecking pipeline; +- one representation of each source rule and function declaration; +- one physical table implementation, union-find, rebuild implementation, and + scheduler; +- optional compile-time decorations for slots and proofs; +- optional evidence storage selected at monomorphization or plan construction, + with a zero-cost `NoEvidence` form; +- no runtime branch that sends a whole program through an independently + maintained semantics. + +The fast native mechanisms do not disappear. Their role changes: they become +the physical lowering of the encoding rather than an independent definition of +surface egglog. + +Likewise, proof production can remain optional without becoming a second path. +A proof-enabled rule is the same normalized rule with a proof plan attached; +it is not a second rule parsed and typechecked in another `EGraph`. + +## A typed encoding contract + +The smallest useful contract is not a generic backend trait. It is a compiler +artifact with four pieces: + +| Piece | Purpose | +| --- | --- | +| Logical program | The actual target-language declarations, rules, actions, and schedules | +| Origin map | Source declaration/rule/action responsible for every generated item | +| Interpretation | How target values, substitutions, and observations map back to the source language | +| Invariants | Facts a physical lowerer may rely on and must preserve, such as canonical views or total slot maps | + +An encoding pass consumes one typed language and produces another typed +language plus this metadata. Its output language must be the next pass's input +language. That makes pass ordering explicit and makes invalid compositions fail +at the compiler boundary rather than in generated source. + +The compiler should preserve administrative operations as typed nodes or typed +annotations long enough for fusion. It must not ask the lowerer to rediscover +them from generated names such as `@UF_Math` or `@AddView`. Name-based +peepholes would retain the entire source generator and introduce a hidden third +path. + +For rules, the conceptual result is: + +```text +normalized rule + + source origin + + optional slot match/action plan + + optional proof skeleton + -> one executable rule plan +``` + +This is a conceptual record, not yet a proposed Rust API. The vertical-slice +experiment should determine which fields are real and which can be derived. + +## Directed composition: slotted first, proofs second + +The project notes already answer the commutativity question: the passes do not +commute; proofs should run after the slotted encoding. That is not a failure of +composition. Composition means that the codomain of the slotted pass is in the +domain of the proof pass: + +```text +E_slot : SlottedEgglog -> CoreEgglog + SlotInterpretation +E_proof : CoreEgglog -> ProofCoreEgglog + ProofInterpretation + +E_both(P) = E_proof(E_slot(P)) +``` + +This order is substantively useful: + +- the slotted pass makes the paper's `beta` and `mp` explicit in rule matching; +- it lowers a union of renamed ids into the base operations that implement it; +- the proof pass then records the actual compiled premises and actions; +- a single rule firing can carry both the slot witness and proof skeleton; +- proof extraction can interpret that firing through both origin maps to name + the original source rule and source-level substitution. + +Arbitrary pass permutation should not be a paper claim. A stronger and more +defensible claim is **typed, directed composition with an interpretation +theorem**. + +### The correctness obligations + +The paper needs to separate four properties that are easy to blur: + +1. **Proof erasure.** Erasing proof decorations from `E_proof(Q)` has the same + observations as running `Q`. +2. **Proof soundness.** Every extracted proof denotes a valid equality in the + semantics of `Q`. +3. **Slotted preservation.** Interpreting the observations of `E_slot(P)` gives + the observations of slotted program `P`. +4. **Physical refinement.** Running the fused physical plan gives the same + observable result as running the literal encoded core program. + +Schematically: + +```text +observe(run_core(E_both(P))) + == observe(run_fused(lower(E_both(P)))) + +decode_slot(erase_proof(observe(run_core(E_both(P))))) + == observe_slotted(P) +``` + +The second equation gives semantic composition. It does **not** by itself give +a pleasant source-level proof. For that, the proof interpretation must map an +encoded rule firing and its renaming witness back to the source slotted rule. +That resugaring/interpretation lemma is a real paper obligation, not metadata +that can be reconstructed after execution. + +## Why the current proof skeleton is the right precedent + +The existing proof design already has a valuable specification/implementation +split: + +- layer 1 describes the proof that would be built as the rule executes; +- layer 2 emits a compact proof skeleton and reconstructs layer 1 later. + +The next engineering step is not to abandon that encoding. It is to stop +storing the skeleton, term identities, and proof nodes as ordinary egglog rows +when the production engine can retain them more compactly. + +In the proposed architecture: + +- layer 1 remains the declarative proof specification used in the paper; +- the typed proof pass derives a skeleton from a normalized rule; +- the fused rule plan stores the static part of that skeleton once; +- a firing records only the dynamic holes that the skeleton needs; +- extraction materializes the existing proof algebra on demand; +- `NoEvidence` erases the skeleton holes and all per-firing writes. + +This is partial evaluation of the proof encoding, not a different proof +semantics. + +## How the literal equality encoding should fuse + +The current term encoding makes several logical objects explicit. A native +lowerer can recognize the typed objects directly and implement them with one +physical structure: + +| Logical encoded object | Fused production representation | +| --- | --- | +| term relation plus canonical view | native constructor/function table, with optional stable `TermId` sidecar | +| explicit per-sort `@UF` | native union-find | +| view collision merge | native congruence/rebuild event | +| occurrence relation/index | native rebuild occurrence index | +| parent/rebuild/cleanup schedule | native commit and rebuild loop | +| `Unit` proof column | erased | +| proof-valued column | compact `CauseId` or receipt sidecar | +| proof-node relations | append-only proof arena materialized on demand | +| generated rule proof | static skeleton plus firing-hole bindings | + +Reaching 5-10% requires essentially all of these fusions. The current +term-only measurements show that frontend cleanup alone is not enough: the +literal relational representation changes row widths, query shapes, write +counts, and maintenance work. + +It is therefore plausible for **encoded semantics with evidence erased** to be +within 5%, because it can lower to nearly the same physical operations as +normal mode. It is not currently plausible for the literal encoded program to +reach that range, nor is there evidence that **always retaining arbitrary proof +evidence** can do so. + +## Slotted-specific implications + +The current slotted rule design is already naturally compiler-shaped: it +computes the paper's `beta` and `mp`, turns each user variable into a leader plus +renaming, and distinguishes fully bound group lookups from genuine +`find-mapping` joins. + +The typed pass should preserve those distinctions. In particular: + +- a known symmetry membership test should stay a lookup, not be expanded into + an enumerating join and then rediscovered by an optimizer; +- extension of `mp` is a real solver operation and should remain explicit; +- union acts on renamed ids, so its origin and renaming witness must survive + into proof interpretation; +- fresh-slot completion must be solved before claiming a complete slotted + encoding; +- the self-edge/group invariant must be stated at phase boundaries, because + the current machinery can expose transient derived facts inside maintenance. + +Some slotted operations may remain relational in the first fused engine. The +architecture does not require every encoding feature to have a native data +structure on day one. It requires one execution plan and an explicit boundary +where a measured hot logical operation can later receive a specialized +physical implementation. + +## Why generic slicing/provenance is not the shared substrate + +The slicing campaign is a useful negative architecture experiment. + +It found that a general recorder plus post-hoc causal reconstruction: + +- added roughly 9,820 lines of provenance recording and 5,300 lines of slicing + including tests, with about +26,182 production lines at PR time; +- still had a witness-free capture floor of 2.213x normal on the decisive Math + experiment; +- required reasoning about row lifetimes, deletes, merge boundaries, + containment, replay identity, and pre-event equality denotation; +- did not become small merely because "any valid support" replaced exact + historical support. + +That does not mean receipts are unusable. It means proof production should not +be implemented as arbitrary execution history followed by a generic graph +query. The proof compiler already knows the rule, its static proof skeleton, +and the exact dynamic holes it needs. Record those holes locally at the rule +and equality-effect boundaries. + +The slicing lesson should become an architectural constraint: + +> No generic recorder, replay engine, or second interpreter may be added to +> support the first proof/slotted vertical slice. + +Slicing can remain out of scope, or later consume an explicitly bounded receipt +interface as a debug feature. It must not define the common runtime substrate. + +## Removing backends changes the paper story + +The older paper pitch used a cross product of expressive features and +performance backends, then claimed that one encoded program was portable over +several backends. If DuckDB, Differential Dataflow, and slicing are being +removed, that claim should be removed rather than simulated by abstractions in +main. + +The replacement story is tighter: + +1. E-graph extensions such as proofs and slotted matching normally cut across + matching, actions, equality, rebuild, extraction, and printing. +2. Expressing each extension as a typed semantic encoding localizes its + definition and makes their order of composition explicit. +3. Literal execution establishes an executable reference semantics. +4. Staging and fusion recover the specialized performance of one production + engine without reintroducing a second language implementation. +5. The implementation is evaluated on semantic parity, composition, + performance, and net production complexity. + +This changes "portability across backends" into **portability of extension +semantics across physical representations**, demonstrated here by a literal +reference execution and one fused execution. If that wording sounds too much +like two backends, omit portability entirely and call the contribution +*composable encodings with semantics-preserving fusion*. + +The paper should not claim that encodings eliminate all extension-specific +engineering. Each encoding still needs a pass, a correctness argument, and +possibly a physical optimization. The claim is that this work is localized and +composes at a declared boundary instead of multiplying through the core. + +## Candidate paper claims and evidence + +| Claim | Required evidence | Current state | +| --- | --- | --- | +| Proofs and slotted semantics are separate encodings | formal definitions plus executable reference translations | proof translation exists; slotted user-rule translation is incomplete | +| The encodings compose | runnable `E_proof(E_slot(P))`, directed composition theorem, source interpretation | not yet demonstrated | +| Fusion preserves the encoding | differential/reference tests plus a physical-refinement argument | absent; proposed work | +| Fusion recovers near-native performance | same-binary literal vs fused vs current-normal benchmarks | absent; current literal term mode is 2.01-2.12x | +| Main becomes simpler | net production LoC, deleted paths, fewer core touchpoints and support gates | absent; must be measured, not asserted | +| Proofs remain independently checkable | existing checker validates source-interpreted composed proofs | checker exists; composed interpretation absent | + +The paper can succeed without 5-10% proof-enabled overhead. A defensible +performance result would report three distinct costs: + +- fused encodings with evidence erased; +- fused proof evidence capture, without extraction; +- proof extraction, simplification, and checking. + +Only the first is the gate for deleting the normal semantic path. Conflating it +with always-on proof capture would make the engineering decision depend on a +much stronger and currently unsupported performance claim. + +## Complexity budget and stop rules + +One engine is not automatically a smaller repository. A typed IR, origin maps, +fusion, and a reference printer can themselves become a large parallel system. +The work should therefore use deletion-backed gates: + +1. **Every production abstraction names the old code it will delete.** A new + pass field or runtime hook is not accepted merely because it may be useful. +2. **No second interpreter.** The reference form is printed from the same typed + artifact and run only by the existing core semantics in tests/artifacts. +3. **No generated-name peepholes.** Fusion operates on typed provenance or + typed operators. +4. **No generic provenance substrate.** The first slice records only holes + demanded by its static proof skeleton. +5. **One vertical slice before broad coverage.** Stop if the slice adds more + production machinery than the old slice it demonstrably replaces. +6. **Keep a running LoC ledger.** Separate production, tests, reference + semantics, and documentation. A smaller core cannot be inferred from a + smaller file count. +7. **Delete as the migration proceeds.** Do not defer all deletion until every + feature is supported; use narrow internal seams so completed families stop + exercising the old path. +8. **Re-measure on every architectural checkpoint.** The slicing campaign + showed that stale cost attribution can steer days of design in the wrong + direction. + +The final deletion gate should require: + +- one frontend/typechecker and one runtime dispatcher; +- one physical equality/rebuild implementation; +- no production execution of printed encoded source; +- no support gate whose only reason is representational inability of the old + generator; +- net production LoC reduction relative to the frozen baseline, or an explicit + maintainer decision that a measured complexity increase is worth the result. + +## Falsifying implementation sequence + +### A0: settle the semantic boundary + +Write down the source and target languages of `E_slot` and `E_proof`, their +observations, pass order, and interpretation maps. Decide whether the composed +proof must name source slotted rules or whether a proof of the compiled core +program is sufficient. + +Stop if the paper team cannot agree on this: the implementation cannot repair +an ambiguous theorem statement. + +### A1: one typed reference artifact + +Change no runtime behavior. Make one tiny constructor/rewrite example produce a +typed encoded artifact from which the current literal core program can be +printed. The artifact must retain source origins without parsing generated +names. + +Gate: printed output behaves exactly like the existing term/proof encoding and +the existing proof checker accepts its proof. + +### A2: fuse one equality/proof vertical slice + +Lower the same artifact directly to the existing native constructor table, +union-find, and rebuild path. Attach one static proof skeleton and record only +its dynamic firing holes under `ProofEvidence`. + +Gates: + +- literal and fused observations match; +- the source-interpreted proof checks; +- `NoEvidence` makes no per-row allocation and adds at most 5% wall time/RSS on + the microcase and a representative existing benchmark; +- the diff includes a named deletion or replacement of the corresponding old + execution branch. + +### A3: measure the proof-capture floor + +Record stable terms and the minimal sound rule/equality receipts, but do not +extract or simplify proofs. This is the optimistic lower bound for proof +availability. + +Do not set 5-10% proof-enabled overhead as a project promise unless this floor +meets it. If it misses, keep evidence optional and proceed with the one-path +design. + +### A4: compose a minimal slotted proof + +Use a program that needs a non-identity renaming and a repeated-variable group +membership check. Run `E_slot` then `E_proof`; compare the literal and fused +forms; extract a proof that names the source rule and carries enough renaming +evidence to check. + +This is the decisive paper slice. Do not begin broad slotted benchmarks until +it works. + +### A5: close known slotted semantic gaps + +Implement and validate fresh-slot completion, settle the phase-boundary group +invariant, and differential-test the translation against the slotted-egraphs +reference implementation. These are correctness gates, not optimization work. + +### A6: migrate semantic families and delete the split + +Move globals/scopes, containers, input, custom merge, delete/subsume, user +indexes, primitives, and extraction one family at a time. Each family must add +parity tests, remove its representation-only rejection, and stop using the old +production path. + +After corpus and proof gates pass, remove the production term/proof execution +mode. Retain only the derived reference printer and paper/test artifacts. + +## Architecture alternatives + +| Alternative | Paper fit | Performance | Complexity outcome | Verdict | +| --- | --- | --- | --- | --- | +| Execute today's literal term encoding universally | strongest superficial dogfooding | current evidence is about 2x term-only and about 3x proofs | deletes native semantics but retains a large generator and maintenance program | reject | +| Typed encodings plus semantics-preserving fusion | keeps encodings and directed composition central | can lower erased mode to current native mechanisms | can delete both independent production paths if deletion gates hold | recommend | +| Native extensible annotation/hook algebra | proofs and slots may share elegant metadata operations | potentially fastest | risks another broad core substrate and weakens the compiler-encoding paper | research alternative, not first slice | +| Paper artifact separate from production main | cleanest immediate main cleanup | production stays fast | paper and engineering may drift; no dogfooding claim | fallback if fusion fails its complexity gate | + +## Open decisions + +1. Must a composed proof name and validate the original slotted rule, or is a + proof over the compiled core rule the paper's theorem? The former is much + more compelling and requires an explicit interpretation lemma. +2. What exactly is `CoreEgglog` for the formalism? It should be small enough to + state semantics, but not chosen as a lowest common denominator for backends + that are being removed. +3. Is the literal printer shipped in main, test-only, or held in the paper + artifact? It must not become a public execution mode by accident. +4. Which stable term identity is genuinely required under `NoEvidence`? Any + always-present identity must earn its measured cost. +5. Can slot and proof plans share a firing substitution without widening every + native binding row? This is a primary performance experiment. +6. What is the accepted production LoC outcome? "Less complexity" needs a + frozen baseline and a measurable deletion target. +7. Is slicing fully out of scope, or a later debug consumer? It should not + influence the first common interface either way. + +## Knowledge-unit map + +| ID | Knowledge unit | Kind | +| --- | --- | --- | +| KU-1 | The current paper direction is proofs plus slotted as composable encodings, with backends and slicing being removed from scope | project decision | +| KU-2 | The intended order is slotted then proofs; the passes do not commute | project decision | +| KU-3 | Literal term-only execution is far outside a 5-10% normal-path gate | measured fact | +| KU-4 | The proof design already separates a declarative layer from emitted skeletons | current design fact | +| KU-5 | The slotted rule translation makes `beta`/`mp` explicit but has unresolved fresh-slot and invariant questions | current design fact | +| KU-6 | General causal recording produced high overhead and a large production diff even after simplification campaigns | measured historical fact | +| KU-7 | Typed fusion can make encoded/no-evidence execution near-native | hypothesis to falsify | +| KU-8 | Composed proof interpretation can recover source-level slotted rules and substitutions | blocked design obligation | +| KU-9 | The architecture will reduce net production complexity | hypothesis to measure | +| KU-10 | Arbitrary proof evidence can always be retained within 5-10% | unsupported stronger hypothesis | + +## Evidence matrix + +| KU | Primary source | Status | Consequence | +| --- | --- | --- | --- | +| KU-1 | project meeting notes, Aug. 5-6; current maintainer direction | Convergent | remove backend portability and slicing from the central architecture | +| KU-2 | project meeting notes lines 102-142 | Convergent | specify typed directed composition, not commutativity | +| KU-3 | `term-encoding-unification.md` fresh same-binary benchmark | Convergent | do not attempt to tune the literal representation to 1.05x | +| KU-4 | `egglog/src/proofs/proof_encoding.md`, proof layers 1 and 2 | Convergent | preserve the semantic layer while changing storage/lowering | +| KU-5 | `slotted-user-rules.md`, fresh-slot gap and open questions | Convergent | composition claims remain gated on slotted correctness work | +| KU-6 | `SLICING-CAMPAIGN-REPORT.md` and its fresh capture-floor experiment | Convergent | prohibit generic recording/replay in the first architecture slice | +| KU-7 | no prototype or benchmark yet | Absent | A2 is a falsifying experiment, not an implementation commitment | +| KU-8 | meeting notes identify resugaring/composition difficulty; no theorem exists | Blocked | A0 must settle the source-level proof contract | +| KU-9 | no fused implementation or deletion diff exists | Absent | use an LoC ledger and named deletion gates | +| KU-10 | current term/proof and slicing capture measurements | Divergent | keep proof evidence optional unless A3 changes the evidence | + +## Source basis + +Highest-authority sources used for this note: + +1. Current maintainer direction in this design discussion: proofs and slotted + remain encodings that compose; alternate backends and slicing are being + removed. +2. `/Users/saul/Downloads/egglog encoding project.md`, especially the Aug. 5-6 + notes on pass ordering, composition, paper claims, and removal of backends + and slicing. +3. [`egglog/src/proofs/proof_encoding.md`](egglog/src/proofs/proof_encoding.md), + the current equality/proof encoding and skeleton design. +4. [`slotted-user-rules.md`](slotted-user-rules.md), the current concrete + slotted user-rule translation and its open semantic gaps. +5. [`term-encoding-unification.md`](term-encoding-unification.md), current-main + benchmark and code-path evidence. +6. `/Users/saul/p/wt/egglog-encoding/pr42-agent-causal-slice-logical-v1/SLICING-CAMPAIGN-REPORT.md`, + the slicing complexity/performance retrospective. + +This note intentionally treats the fusion architecture, its performance, its +net LoC effect, and source-level composed-proof interpretation as proposals. +They are not established by the current sources. + +## Review checklist + +- Does the paper team agree that an encoding is the logical pass, not a mandate + to execute its literal output? +- Is directed `E_proof(E_slot(P))` the intended meaning of composition? +- Is source-level proof interpretation required? +- Are backends and slicing definitively out of the central claims? +- Does A2 delete a named old path before any broad framework is built? +- Are disabled, capture-only, extraction, and checking costs reported + separately? +- Does every complexity claim have an LoC/touchpoint measurement? diff --git a/incremental-unification-pr-roadmap.md b/incremental-unification-pr-roadmap.md new file mode 100644 index 00000000..1222d3d1 --- /dev/null +++ b/incremental-unification-pr-roadmap.md @@ -0,0 +1,615 @@ +# Incremental PR roadmap to one engine + +- Status: proposed sequence; no runtime implementation has started +- Date: 2026-08-12 +- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` +- Architecture companions: [`term-encoding-unification.md`](term-encoding-unification.md) + and [`encoding-architecture-bridge.md`](encoding-architecture-bridge.md) +- Current-main performance companion: + [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) + +## Outcome + +There is a credible incremental route to one production engine with optional +proof evidence. It should not begin by replacing the native union-find with +today's ordinary-table `@UF` encoding. It should proceed in this order: + +1. remove the second generated-program frontend; +2. erase proof-only data from the evidence-disabled plan; +3. make the evidence-disabled encoded operations lower to the existing native + tables, relational union-find, and rebuild driver; +4. delete term-only as an independently executed production mode; +5. move proof evidence onto optional sidecars of those same operations; +6. delete the remaining source-generated proof executor one semantic family at + a time. + +The destination has one union-find, not a native UF plus a relational UF: + +```text +logical Equivalence operation + | + v +EquivalenceTable one Table implementation + parents: UnionFind the only canonicalizer + displaced: (child, epoch) relational change stream + reasons?: (left, right, cause, epoch) optional proof sidecar +``` + +`DisplacedTable` already establishes the important precedent: it is relational +at the database boundary and specialized underneath. A proof reason sidecar is +not a second equivalence structure. It explains effective unions made by the +one structure. + +The 5-10% target applies first to **evidence erased**. Full proof capture, +extraction, simplification, and checking must be measured separately. One +engine does not require proofs to be always recorded. + +The current-main decomposition shows that the performance work cannot be one +serial "optimize UF" campaign. Math is maintenance-dominated, Pointer is +frontend-dominated, and Luminal is dominated by transformed user-rule +planning/search plus the generated frontend. The dependency order below still +removes the generated frontend before deleting an executor, but focused +performance PRs should proceed against the discriminator for their own cost +family rather than treating suite aggregate as one mechanism. + +## What the two session investigations change + +### The `single codebase` Claude session + +The local Claude session +`7fd2857d-167e-48c1-9f0c-c3c5f42f97c6` correctly identifies the central +reframe: + +- keep the encoding as the definition of logical semantics; +- treat specialized tables, union-find, and rebuild as a certified physical + implementation of that encoding; +- do not equate "encoded" with "execute every administrative relation and + maintenance rule literally"; +- use a literal form as a differential oracle and paper artifact rather than a + separately maintained production engine. + +It also identifies useful current costs: term relations and `mint-*` have no +proofs-off reader, `Unit` payloads are dead in term-only mode, and the ordinary +table `@UF` duplicates the existing relational `DisplacedTable` facade. + +However, the session's claimed pure-encoding floor of roughly 1.3-1.6x is not a +measurement. Its own adversarial critique calls that number a prior, observes +that Luminal's transformed-search regression had not been explained, and shows +that some proposed pre-frontend performance gates were arithmetically +unreachable while the second frontend remained. This roadmap therefore uses +the session for hypotheses and architecture, not as proof of a floor. + +### The Luminal-overhead Codex session + +The Codex session +`019ff6c4-f93d-70e0-9440-c2f3e97bc4fa` supplies two decisive corrections. + +First, native Luminal's original "outside rulesets" number was partly an +accounting bug. PR +[#61](https://github.com/saulshanabrook/egglog-encoding/pull/61) records direct +experimental scheduler runs. Its six-round comparison left wall time unchanged +while changing the report from 6 to 16 recorded rulesets and reducing the +unattributed share from 97.39% to 55.45%. + +Second, the remaining proof/term frontend cost is real and measured: + +| Luminal typechecking component | Term-only | Proof generation | +| --- | ---: | ---: | +| All generated typechecking | about 187 ms | about 421 ms | +| Standalone actions and lets | about 104 ms | about 283 ms | +| Rules | about 66 ms | about 120 ms | +| Constraint solving | about 52 ms | about 135 ms | +| Primitive overload validation | about 32 ms | about 86 ms | + +Source typechecking itself was only about 34 ms. The generated program's 1,634 +large top-level lets accounted for about 69% of proof-mode typechecking and its +491 rules for another 24%. About 48% of typechecking leaf CPU was allocator or +memory-library work; `PrimitiveWithId::accept` appeared below about 20.5% of +the samples. + +This changes the PR order. The first architectural performance PR should emit +typed generated actions, not optimize the UF. It both removes measured cost +and collapses a duplicated compiler path. Optimizing the general constraint +solver first would tune a path that the typed lowering is intended to delete. + +Proof extraction has a separate small win: actual Luminal spent about 54 ms +gathering 1,634 globals while constructing `ProofStore`, then about 53 ms +gathering them again during `remove_globals`. Reusing that map should remove +one scan. The eggcc fixture instead needs later work on its large proof DAG; +its dominant extraction cost was proof-store conversion, not globals. + +## Sequencing rules + +Every production PR in the critical path should satisfy these rules: + +1. **Name a deletion.** A new type or hook must identify the old parse, + generated relation, maintenance rule, runtime branch, or proof row it + replaces. +2. **Improve a live mode.** Except for the already-open accounting prerequisite + and one paper-composition checkpoint, every PR must either produce a + statistically supported runtime/RSS improvement or delete a production + execution branch with identical performance. +3. **Keep exact mode labels.** Measure `off`, `term`, `proofs`, + `proof-extraction`, and `proof-testing` separately. `proofs` means capture + without automatic extraction or validation. +4. **One variable per benchmark.** Compare the same binary protocol, fixture, + thread count, rounds, and report schema. Preserve anomalous samples. +5. **No generated-name peepholes.** Fusion consumes typed operations or origin + metadata, never names such as `@UF_Math`. +6. **No generic recorder.** Proof capture records only the dynamic holes of a + static proof skeleton. The causal-slicing recorder's optimistic Math floor + was already 2.213x and its campaign added a large amount of code. +7. **Protect the disabled path.** Any proof infrastructure that makes `off` + measurably slower has the wrong layout. A disabled policy should allocate + nothing and should not add dynamic dispatch to row or union hot paths. +8. **Recheck parallelism before deletion.** The current evidence is + single-threaded. The final evidence-erased path must preserve correctness + and performance under the supported multithreaded configuration. + +Each performance PR should carry a small ledger in its description: + +| Item | Required entry | +| --- | --- | +| Baseline | exact base and candidate SHAs | +| Modes | exact treatment names | +| Workloads | focused discriminator plus six-file suite | +| Effect | wall ratio, RSS ratio, and relevant phase/ruleset delta | +| Semantics | proof/corpus parity result | +| Complexity | production lines added, replaced, and deleted | +| Decision | proceed, revise attribution, or stop | + +## Recommended PR queue + +The queue is deliberately front-loaded with changes that improve today's +encoded proof path even if the later fusion design changes. + +| PR | Scope | Current overhead removed | Named deletion or replacement | +| --- | --- | --- | --- | +| P0a | Merge ruleset-accounting PR #61 (complete in `5ead0a0`) | none; makes direct scheduler attribution sound | old outer-only report accumulation | +| P0b | Report rule assembly and disjoint frontend/command stages | none; prevents the remaining residual from being misdiagnosed | wall-minus-rules inference as the primary diagnosis | +| P1 | Reuse proof globals during extraction | one roughly 53 ms Luminal global scan | second `gather_globals` traversal | +| P2 | Emit typed top-level generated actions | largest measured Luminal generated-typecheck bucket | action string generation, reparse, re-desugar, re-typecheck, and generated-global removal for that family | +| P3 | Emit typed generated rules, facts, and schedules | second-largest generated-typecheck bucket and repeated plan setup | corresponding source-string and untyped-command path | +| P4 | Emit typed declarations/merges and remove the second frontend | remaining generated parsing/typechecking and cloned-e-graph bookkeeping | cloned `original_typechecking` `EGraph` and the generated-program frontend loop | +| P5 | Erase proof-only storage under `NoEvidence` | term-row double writes, mint calls, and dead payload width | proofs-off term relations, `mint-*`, and `Unit` proof columns | +| P6 | Make evidence-erased functions/globals identity-lowered | Luminal program/query expansion and plan-construction work | duplicate views, rewritten user queries, and nullary-global view expansion under `NoEvidence` | +| P7 | Route evidence-erased equality through one relational UF | generic-table UF merges and about 43-48 ms Math path compression | per-sort ordinary `@UF` merge programs and `@parent` rules under `NoEvidence` | +| P8 | Route evidence-erased canonicalization through fused rebuild; retire term-only mode | about 382-397 ms Math encoded rebuild plus schedule overhead | generated rebuild/cleanup/subsume schedules and the independently executed term treatment | +| P9 | Add proof reasons to the same `EquivalenceTable` and migrate equality proofs | proof-valued UF rows, eager path proof composition | proof-mode ordinary `@UF`, compression proof rules, and equality proof-node rows | +| P10 | Replace proof term relations with an immutable `TermArena` | append-only per-derivation term rows and mint traffic | constructor/custom term relations and their mint primitives in proof mode | +| P11 | Attach static proof skeletons to the one typed rule plan | duplicated proof-instrumented rules and dynamic proof-node construction | proof rule copy and most proof-node relations | +| P12 | Migrate remaining semantic families and delete the source proof executor | family-specific encoded maintenance and support gates | `ProofInstrumentor` production execution, `proof_check_program`, and representation-only rejections | + +P0 and a later minimal slotted-plus-proof composition test are the only planned +PRs that do not directly reduce overhead or delete an execution branch. + +## PR details and gates + +### P0a: merge accurate direct-ruleset accounting + +PR #61 merged in `5ead0a0cacf847a129294a870de13503f2d7f9c4`. It is an +observability prerequisite, not a performance result. + +The corrected Luminal benchmark is the canary: wall time should remain +statistically unchanged while all 16 native rulesets remain visible and no +ordinary schedule is double-counted. + +### P0b: make the remaining overhead additive + +Add per-ruleset Assembly before Search and split outside-ruleset time into +exclusive leaves under Lowering (Parse, total Typecheck, Other) and Commands +(Install, Actions/input, Other/schedules), plus a derived residual. Total +Typecheck intentionally combines source and generated checking so the +off-versus-encoded delta answers how much checking the encoding added. The +source pass performed in the separate original-typechecker e-graph must still +be charged to that outer total rather than leaking into Lowering / Other. + +The acceptance artifact is the additive six-file table in +[`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md): +the named buckets plus residual must reconstruct process wall time, and the +instrumentation itself must have no detected material tax. Keep this PR scoped +to timing; do not combine it with the shared-globals behavior change from the +larger phase-timer prototype. + +### P1: reuse globals during proof extraction + +Compute the globals environment once for the requested proof and pass or own it +through proof-store construction and global removal. The ownership boundary +should make staleness impossible; this should not become a general cache keyed +by program identity. + +Gate: + +- actual Luminal `proof-extraction` loses one roughly 50 ms scan; +- `proofs` is unchanged, because it performs no extraction; +- eggcc proof-extraction does not regress; +- all proof snapshots and strict proof tests pass. + +This is a useful warm-up but is not on the core unification dependency chain. + +### P2: typed top-level actions first + +Introduce the smallest private generated-program builder needed to construct +resolved variables, calls, expressions, and `ResolvedNCommand::CoreActions`. +Keep declarations on the old path temporarily, but let declaration processing +return the `FuncType` and resolved primitive handles the typed action builder +needs. + +Do not introduce a public general-purpose IR yet. A transitional mixed command +enum is acceptable only if P4 names and deletes it. The invariant owned by the +builder is meaningful: every generated declaration is registered exactly once, +and every emitted call refers to that registered typed object. + +Gate: + +- the 1,634-action Luminal category no longer enters the general constraint + typechecker a second time; +- the proof-generation wall-time CI shows a real improvement; +- term-only also improves; +- emitted behavior and proof propositions are unchanged; +- the PR deletes the old top-level action parse path at its converted call + sites. + +The observed bucket suggests a large win, but no percentage should be promised +until the branch is measured. If the improvement is much smaller than the +removed 104/283 ms typecheck buckets, profile before P3 rather than optimizing +the constraint solver by assumption. + +### P3: typed rules, facts, and schedules + +Construct `ResolvedRule`, resolved facts, and resolved schedules directly. +Preserve rule name, ruleset, evaluation mode, `no_decomp`, +`include_subsumed`, source origin, and the existing proof-head/skeleton layout. +Continue to feed the existing typed-to-core rule machinery so groundedness, +canonicalization, duplicate-variable removal, and rule installation are not +reimplemented. + +Gate: + +- the 491-rule Luminal category no longer runs the second general typechecker; +- generated rule typechecking falls by approximately the measured category, + subject to a fresh profile; +- rule plans and canonical database results match; +- the converted rule/fact/schedule string builders are deleted. + +### P4: typed declarations and deletion of the second frontend + +Finish typed emission for sorts, functions, indexes, merge bodies, and +remaining commands. Separate source type information from target declaration +registration, but do not retain a cloned `EGraph` just to hold the source type +environment. + +At the end, the pipeline should be: + +```text +source parse/desugar/typecheck once + -> typed encoding artifact + -> typed declarations/rules/actions registered and run directly +``` + +Delete the loop in `EGraph::resolve_command` that turns each generated command +back into an unresolved command, desugars it, typechecks it, and removes globals +again. Delete the cloned typechecking-e-graph chain once callers use the typed +artifact. + +Gate: + +- no generated command is reparsed or passed through the general typechecker; +- normal source diagnostics remain source-oriented; +- Luminal and eggcc frontend time falls materially; +- the full support/proof corpus and `make check` pass; +- net production LoC for the frontend split is down, not merely moved. + +### P5: evidence-erased storage diet + +Make proof evidence an explicit compile-time plan policy. Under `NoEvidence`: + +- do not declare or populate immutable term-node relations; +- do not register or call `mint-*` row primitives; +- do not add a `Unit` proof column to views or UF relations; +- let a view/table allocate its default e-class with the same `FreshId` + mechanism used by native constructors. + +Under `ProofEvidence`, preserve current behavior until P9/P10 replace it. This +temporary policy split is acceptable because it is a staged erasure in one +typed artifact, not two source frontends. + +Focused gates: + +- Herbie and Hardboiled wall/RSS, where per-firing build cost should be visible; +- eggcc RSS and total term rows; +- Luminal user-rule search, to test whether row/schema width contributes to its + expansion cost; +- no change to proof generation or checking. + +If build-heavy workloads barely move, revise the overhead ledger before adding +new storage features. + +### P6: evidence-erased identity lowering + +Under `NoEvidence`, one source function should use one physical function table. +The logical term relation and canonical view may remain distinct in the typed +reference artifact, but fusion maps them to the same table. User rule bodies +must therefore keep the original narrow atom shape. Source globals should use +the common `remove_globals` lowering rather than gaining an additional term +relation, FD view, index, and rebuild rule. + +This is the decisive Luminal PR. The current transformed user search is about +475-477 ms versus about 5.3-5.7 ms in `off`, while generated maintenance is only +a few milliseconds. A successful identity lowering should make the same user +rules compile to the same physical query shape. + +Gate: + +- compare ruleset-by-ruleset search, not only whole wall time; +- the evidence-erased user rules are physically shape-equivalent to normal; +- generated rule/index counts for globals disappear; +- if Luminal user search remains more than 2x native, stop and inspect the + remaining plan difference before touching the UF. + +### P7: one relational UF for evidence-erased execution + +Introduce a typed `Equivalence` operation and lower it to the existing +`DisplacedTable`/`UnionFind` implementation. The database continues to +see a table-shaped interface and a displaced-value change stream. The encoder +no longer emits an interpreted ordering merge or a path-compression rule under +`NoEvidence`. + +This PR should prepare, but not prematurely implement, the proof contract: +`union(left, right, optional cause)` and an effective-union event. It must not +add a second parent forest or allocate reasons when evidence is disabled. + +Gate: + +- `@parent` disappears from evidence-erased reports; +- Math improves from removal of its roughly 43-48 ms path-compression ruleset + plus interpreted UF merge overhead; +- min-id leader, push/pop, extraction, and update semantics match; +- off-mode code generation and performance remain unchanged. + +### P8: fused rebuild and retirement of term-only execution + +Lower the typed canonicalization operation to the existing container-first +bulk rebuild/fixpoint driver. Under `NoEvidence`, delete generated occurrence +declarations, row-rewrite rules, cleanup/subsume schedules, and trailing +maintenance schedules. The literal typed artifact can still print those +logical operations for the paper or differential oracle. + +At this point evidence-erased encoding should be physically identical to the +current normal path. Make `term` an alias temporarily only if needed to prove +that identity, then delete it as a production treatment and CLI execution +choice. Retain a test-only literal interpreter configuration if it pays for its +maintenance through the differential oracle. + +Deletion gate: + +- evidence-erased wall and RSS are within 1.05x on every suite file and the + aggregate CI contains 1.0; +- canonical database parity holds for the corpus; +- multithreaded parity and performance pass; +- no production command dispatcher chooses between normal and term-only + programs; +- the PR deletes more production path code than it adds. + +If this gate fails despite supposedly identical physical operations, another +hidden path remains. Do not loosen the target to bless it. + +### P9: proof-capable relational UF, without a second UF + +Extend the same `EquivalenceTable` with a `ProofEvidence` policy. On an +effective union, append a compact reason event naming the two pre-union leaders, +the chosen leader, a `CauseId`, and the epoch. Canonical `find` still consults +the one native UF. Path compression is a storage optimization and does not +create proof nodes. + +Reconstruct equality explanations lazily from the reason forest and lower them +to the existing `ProofStore`/checker format. The first acceptance case is: + +```text +insert a, b, f(a), f(b) +record a = b by a source rule or fiat +derive f(a) = f(b) by congruence +materialize the explanation +accept it with the current independent checker +``` + +Then migrate the current proof `@UF` users and delete proof-valued parent rows, +`Trans`/`Sym` nodes created solely for compression, and the encoded parent +rules. + +Gate: + +- one physical `UnionFind` exists; +- `NoEvidence` remains binary/performance neutral; +- all equality and congruence proof fixtures validate; +- proof-generation Math improves before proceeding to term storage. + +### P10: immutable terms as an arena, not ordinary rows + +Proofs need stable syntactic identity after e-classes move or rows are deleted. +They do not require one append-only database row per derivation attempt. Add an +immutable, hash-consed `TermArena` used only by `ProofEvidence`: + +```text +TermId -> constructor and child TermIds +row/eclass -> witness TermId +CauseId -> source rule, merge, congruence, or fiat receipt +``` + +Move constructor and custom-row proof reconstruction onto this arena, then +delete proof-mode term relations and `mint-*` primitives. + +Gate on proof-generation and proof-extraction separately. Eggcc wall/RSS is the +important discriminator because its large proof DAG, not global scanning, +dominates extraction. + +### P11: one rule plan with an optional proof skeleton + +Preserve the current good idea: the static proof shape is known when a typed +rule is compiled. Attach that skeleton and source origin to the one normalized +rule plan. Under `ProofEvidence`, a successful firing records only the stable +row/term/cause IDs needed to fill its holes. Under `NoEvidence`, the plan has no +holes and emits no receipt. + +This is deliberately not a general causal journal. It does not record arbitrary +history and later search for a proof; the proof compiler specifies exactly +which dynamic values are needed. + +Delete the duplicate proof-instrumented rule, proof-node relations replaced by +the skeleton/receipt pair, and eventually the duplicated command stream used +only by checking. + +Gate: + +- exact proof propositions check even if pretty-printed proof shape changes; +- full eggcc 2mm proof-validating performance is reported distinctly from + capture-only and extraction-only results; +- disabled execution remains unchanged; +- net production LoC trends downward against the frozen encoder baseline. + +### P12: family-by-family migration and final deletion + +Do not put the long tail in one PR. Use separate deletion-backed PRs for: + +1. custom functions and merge bodies; +2. containers and normalization receipts; +3. input, globals, scopes, push/pop, and the Rust API; +4. delete, subsume, user indexes, and extraction; +5. remaining primitive and tuple-output cases. + +Each family PR must remove its old encoder branch and at least one +representation-only unsupported reason. Once the corpus and proof gates pass, +delete production execution through `ProofInstrumentor`, +`proof_check_program`, the cloned proof program, and the proof/term dispatcher. + +The surviving proof assets should be the proof algebra, `ProofStore`, +simplifier, extractor/materializer, independent checker, typed skeletons, +origin maps, and the derived reference printer. + +## Slotted composition checkpoint + +After P11 proves that one typed rule plan can carry a source origin and proof +skeleton, add a narrow paper checkpoint before broad P12 migration: + +1. lower one slotted rule requiring a non-identity renaming; +2. run the proof pass after the slotted pass; +3. execute the fused plan; +4. extract a proof whose interpretation names the source slotted rule and + substitution; +5. compare it with the literal typed encoding and validate it. + +This PR may not improve runtime. Its purpose is to prevent a fast proof-only +architecture from invalidating the paper's actual composition claim. It should +not grow a second interpreter or generic provenance framework. + +## Why this order is preferable to the alternatives + +### Do not start with the UF + +UF work is important for Math, but it cannot explain Luminal's dominant +transformed user search or the generated frontend. Starting there would improve +one component while leaving the two largest cross-workload sources intact. + +### Do not start by optimizing the constraint solver + +The solver and primitive overload validation are hot, but almost all of their +proof/term delta comes from typechecking generated commands a second time. Typed +emission removes that work and reduces code. Solver caching is a fallback only +if source-program typechecking remains material afterward. + +### Do not tune the literal source encoding all the way to 10% + +P5 is a useful measured erasure experiment. P6-P8 intentionally stop treating +the literal tables and maintenance rules as the production representation. +Specialized relational storage is not a betrayal of the encoding; it is the +physical lowering that makes the encoding viable. + +### Do not revive the slicing recorder for proofs + +The slicing campaign showed both the code and capture cost of a broad execution +journal. P9-P11 instead record local, typed causes and only the holes required +by known proof skeletons. + +## Complexity ledger + +Freeze these current baselines before P2: + +- encoder-facing production modules + (`proof_encoding*`, `proof_head`, `proof_fresh`, and + `proof_container_rebuild`): about 6,999 lines; +- the full `egglog/src/proofs` production directory excluding + `proof_tests.rs`: about 11,949 lines; +- `DisplacedTable`: 517 lines, much of which remains as the one physical UF; +- current unsupported-reason count and excluded corpus files; +- command-dispatch branches and benchmark/test treatments. + +Do not score moved native UF/rebuild code as deletion merely because it gets a +more general name. The meaningful complexity wins are: + +- one source typecheck and one target registration path; +- one physical function table per source function when evidence is erased; +- one physical UF and rebuild implementation; +- one normalized rule plan; +- no production execution of generated source; +- fewer support gates and test-matrix axes; +- net production LoC reduction by the final P12 gate. + +## Stop rules + +1. If P2 does not recover a substantial portion of the measured generated + action typecheck bucket, stop and re-profile before P3. +2. If P6 does not collapse Luminal's transformed user search, do not infer that + UF or rebuild work will rescue the 10% target. +3. If P8 cannot make evidence-erased execution physically and measurably + equivalent to normal, do not delete the normal dispatcher. +4. If P9-P11 proof capture misses the accepted proof-enabled gate, keep + evidence optional. This does not invalidate the one-engine design. +5. If any evidence hook taxes `off`, move the policy choice to plan + construction/monomorphization or a separate build; do not accept a permanent + 5-10% tax merely because it is inside the target band. +6. If a new abstraction grows faster than the old family it replaces, stop the + broad migration and retain the typed reference artifact plus current native + lowering. + +## Recommended immediate next action + +Land **P0b, additive phase reporting**, next. Then make **P2, typed top-level +action emission**, the first architectural PR. P1 can land independently as a +small extraction cleanup. + +P2 is the best first commitment because it is simultaneously: + +- supported by a clean profile; +- useful to term and proof modes; +- a reduction in compiler duplication; +- required by any typed encoding/fusion paper story; +- independent of the unresolved proof-storage design; +- a way to establish the builder and measurement discipline needed for every + later PR. + +In parallel, use eggcc as the discriminator for a narrowly scoped +ruleset-assembly experiment: generated `@rebuilding`/`@parent` assembly is +about 253 ms there. Do not assume typed emission alone removes that +per-invocation assembly cost. + +Only after P2-P4 should the project choose exact Rust APIs for the relational +UF sidecar. That keeps the UF design grounded in the typed artifact that will +actually call it, instead of preserving assumptions forced by today's +generated source schema. + +## Source basis + +- Current source at the pinned baseline, especially `EGraph::resolve_command`, + `ProofInstrumentor`, `DisplacedTable`, the bridge rebuild driver, and the + typed/core rule lowering. +- Local Claude session `7fd2857d-167e-48c1-9f0c-c3c5f42f97c6`, titled + `single codebase`, including its research workflow output + `wvkzpajc3.output`. +- Local Codex session `019ff6c4-f93d-70e0-9440-c2f3e97bc4fa`, including the + Luminal phase/typechecking profile and PR #61. +- `/tmp/term-encoding-always-on-bd4752e.jsonl`, the same-binary current-main + benchmark cache described in `term-encoding-unification.md`. +- `/Users/saul/Downloads/egglog encoding project.md`, for the paper's directed + slotted-then-proof composition goal. +- `/Users/saul/p/wt/egglog-encoding/pr42-agent-causal-slice-logical-v1/SLICING-CAMPAIGN-REPORT.md`, + for the generic-recorder complexity and capture-floor stop evidence. diff --git a/term-encoding-overhead-breakdown.md b/term-encoding-overhead-breakdown.md new file mode 100644 index 00000000..3bb610d3 --- /dev/null +++ b/term-encoding-overhead-breakdown.md @@ -0,0 +1,577 @@ +# Where Current Term-Encoding Time Goes + +Status: measured diagnostic on current `origin/main`, followed by the retained +V3 phase-timing implementation documented in the evidence ledger below. + +Baseline commit: `5ead0a0cacf847a129294a870de13503f2d7f9c4` + +This commit includes PR #61, which records direct ruleset runs in the overall +report. That fix is necessary, but it does not account for rule planning and +assembly or for the frontend pipeline outside ruleset execution. + +## Short answer + +There is no single dominant term-encoding cost across the benchmark suite. + +- Math is dominated by replacing native rebuilding with `@rebuilding` and + `@parent` plus the extra Apply/Merge work of the encoded representation. +- Pointer analysis is dominated by generating, desugaring, typechecking, and + installing the encoded program. Its actual rule execution is tiny. +- Hardboiled is split between the generated frontend and slower transformed + user rules. +- Luminal is dominated by two nearly equal costs: generated frontend work and + planning/searching transformed user rules. Encoded UF/rebuild maintenance is + only about 5.5% of its slowdown. +- Herbie is mixed across frontend, transformed user rules, encoded + maintenance, and command work. +- eggcc is also mixed. A previously hidden ruleset-assembly cost is important: + assembling `@rebuilding` and `@parent` costs about 253 ms, before their + Search/Apply/Merge timers begin. + +Therefore the current two-part model is incomplete: + +1. encoded UF and rebuild maintenance; +2. generating and re-typechecking an encoded program; + +There is a third material cost: + +3. changed physical rule shape, including per-invocation rule assembly, + query planning, and user-rule search. + +There is also smaller workload-dependent top-level command work. + +## Question and hypotheses + +Question: for each of the six representative workloads, which mechanisms +explain the wall-time increase from `off` to `term` on the same binary? + +The diagnostic distinguished these competing hypotheses: + +- H1: explicit encoded UF/rebuild maintenance dominates. +- H2: the second frontend and generated-program installation dominate. +- H3: source rules become physically different queries and spend more time in + rule assembly/planning/search even when generated maintenance is cheap. +- H4: top-level actions, input loading, checks, extraction, or scheduler-driver + work dominate outside the rulesets. + +The results support different hypotheses on different workloads. + +## Measurement + +The decisive report is: + +```text +/tmp/term-overhead-main-5ead0a0-instrumented-assembly-v1.jsonl +``` + +The instrumented binary SHA-256 is: + +```text +3c140fc59901ec0d448778dc511f23ed12498434344ebd74010fe4db75dcd48e +``` + +Command shape: + +```text +./bench.py \ + --target . --treatment term \ + --compare-target . --compare-treatment off \ + --rounds 6 --timeout-sec 120 --force-run \ + --report /tmp/term-overhead-main-5ead0a0-instrumented-assembly-v1.jsonl +``` + +Both endpoints use the same release binary, workload bytes, fact-directory +bytes, and one execution thread. Runs alternate `off` and `term` for each file. +All 72 runs succeeded. + +The instrumentation measured disjoint intervals for: + +- initialization and source-file reading; +- source parsing, macros, typechecking, and other source resolution; +- encoding generation, including parsing emitted encoding text; +- generated desugaring and generated typechecking; +- installing functions and compiled rules; +- top-level actions, input, scheduler-driver work, and other commands; +- per-ruleset assembly, Search, Apply, unattributed execution, Merge, and + Rebuild. + +The report's additive residual is wall time minus all those intervals. It is +only 0.5% to 6.2% of each measured slowdown, which is a useful check that the +phase boundaries explain nearly all of the difference. + +An earlier stock-main report, without the temporary extra timers, is retained +at: + +```text +/tmp/term-overhead-main-5ead0a0-v1.jsonl +``` + +Its ratios agree with the diagnostic run. A separate ten-round diagnostic run +at `/tmp/term-overhead-main-5ead0a0-instrumented-v1.jsonl` contains substantial +machine-contention outliers and is retained rather than filtered. During the +campaign an unrelated long-running process was executing +`/tmp/churchroad-wide-multiply.egg`. The decisive six-round paired run was +stable despite that background process; its tight paired confidence intervals +support the relative decomposition, while its absolute milliseconds remain +machine-specific. + +## Additive slowdown decomposition + +All numbers are paired mean `term - off` wall milliseconds over six rounds. +Percentages are shares of that file's wall-time increase. + +| Workload | Off | Term | Slowdown | Frontend | Transformed user rules | UF/rebuild substitution | Command work | Residual | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Math | 376.6 | 839.8 | 463.2 | 3.4 (0.7%) | 166.8 (36.0%) | 289.2 (62.4%) | 1.1 (0.2%) | 2.7 (0.6%) | +| eggcc | 839.5 | 1235.2 | 395.7 | 142.3 (36.0%) | 118.8 (30.0%) | 87.4 (22.1%) | 37.6 (9.5%) | 9.7 (2.4%) | +| Pointer | 8.0 | 15.5 | 7.6 | 6.4 (83.9%) | 0.2 (2.2%) | 0.6 (7.3%) | 0.0 (0.5%) | 0.5 (6.2%) | +| Hardboiled | 115.6 | 222.1 | 106.5 | 49.4 (46.4%) | 35.2 (33.1%) | 11.5 (10.8%) | 6.9 (6.5%) | 3.4 (3.2%) | +| Luminal | 372.5 | 1291.0 | 918.5 | 357.8 (39.0%) | 418.1 (45.5%) | 50.5 (5.5%) | 68.2 (7.4%) | 24.0 (2.6%) | +| Herbie | 54.3 | 109.5 | 55.2 | 17.5 (31.8%) | 14.7 (26.7%) | 12.0 (21.7%) | 9.4 (17.1%) | 1.5 (2.7%) | + +The paired 95% confidence intervals for the total slowdowns are: + +| Workload | Paired slowdown 95% CI | +| --- | ---: | +| Math | 451.4–475.1 ms | +| eggcc | 384.8–406.7 ms | +| Pointer | 7.2–7.9 ms | +| Hardboiled | 103.4–109.6 ms | +| Luminal | 889.1–947.9 ms | +| Herbie | 53.7–56.7 ms | + +### Category definitions + +The table is exactly additive. + +- Frontend includes initialization, source-file reading, source parsing, + macros, source typechecking, other source resolution, encoding generation, + generated desugaring, generated typechecking, and installing functions and + rules. +- Transformed user rules includes Assembly, Search, Apply, Execution overhead, + and Merge for rulesets whose names do not begin with `@`. Rebuild is excluded + from this column. +- UF/rebuild substitution includes every phase of generated `@...` maintenance + rules plus the `term - off` difference in non-`@` Rebuild. This subtracts the + native rebuilding that term mode replaces, rather than presenting generated + maintenance as if native rebuilding were free. +- Command work includes top-level actions, input, schedule interpretation after + subtracting all recorded ruleset phases, and checks/extraction/other commands. +- Residual is process wall time not claimed by any measured phase. + +## What the rule timers say + +### Math: encoded maintenance really is the main problem + +Term mode spends 443.0 ms in generated `@...` maintenance: + +- 300.7 ms Search; +- 63.3 ms Apply; +- 78.5 ms Merge; +- less than 0.5 ms assembly and unattributed work. + +Off mode spends 153.7 ms in native Rebuild. Replacing that native work therefore +costs a net 289.2 ms. The transformed default ruleset adds another 88.0 ms Apply +and 76.8 ms Merge. Generated typechecking is only 1.7 ms. + +This is the workload where optimizing or native-backing the encoded UF/rebuild +machinery is directly decisive. + +### eggcc: rule assembly was hiding outside the report + +Term mode's generated maintenance costs 339.8 ms, but 253.0 ms of that is +ruleset assembly before Search starts: + +- `@rebuilding` assembly: 206.8 ms; +- `@parent` assembly: 43.4 ms; +- other generated maintenance assembly: about 2.7 ms. + +Off mode spends 252.5 ms in native Rebuild, leaving a net UF/rebuild +substitution cost of 87.4 ms. + +The frontend adds 142.3 ms: + +- encoding generation: 37.0 ms; +- generated desugaring: 13.1 ms; +- generated typechecking: 71.0 ms; +- installation: 23.6 ms; +- small offsets in the unchanged source frontend: about -2.4 ms. + +Transformed user rules add 118.8 ms, including 57.6 ms Search, 26.6 ms Apply, +21.1 ms Merge, and 13.0 ms assembly. Schedule-driver work adds another 36.6 ms. + +So neither "UF" nor "re-typechecking" alone explains eggcc. Avoiding repeated +assembly of the large generated maintenance rulesets is also a first-class +opportunity. + +### Pointer: almost entirely the second frontend + +The total slowdown is only 7.6 ms, but 6.4 ms is frontend work: + +- encoding generation: 2.3 ms; +- generated desugaring: 0.5 ms; +- generated typechecking: 2.3 ms; +- installation: 1.2 ms. + +User-rule execution adds about 0.2 ms and the net maintenance substitution adds +about 0.6 ms. This benchmark primarily measures fixed per-program encoding +cost, not UF execution. + +### Hardboiled: frontend first, changed user search second + +The frontend contributes 49.4 ms, led by 25.8 ms generated typechecking and +12.1 ms encoding generation. Transformed user rules add 35.2 ms, including +24.6 ms Search. Net encoded maintenance is 11.5 ms. + +### Luminal: not a UF bottleneck + +The 918.5 ms slowdown divides primarily into: + +- 357.8 ms frontend; +- 418.1 ms transformed user rules; +- 68.2 ms command work; +- only 50.5 ms net UF/rebuild substitution. + +The frontend includes 192.0 ms generated typechecking, 88.2 ms encoding, +43.7 ms installation, and 25.4 ms generated desugaring. Top-level encoded +actions add 65.0 ms. + +Within transformed user rules, Search adds 322.3 ms and assembly adds 90.9 ms. +This agrees with the corrected ruleset report: `fusion_grow` and `fusion_pair` +search much slower under the wider encoded relation/query shapes. Improving +`@UF` alone cannot materially close Luminal's gap. + +### Herbie: no single dominant bucket + +Frontend contributes 17.5 ms, transformed user rules 14.7 ms, net maintenance +12.0 ms, and command work 9.4 ms. The command bucket includes about 7.7 ms in +checks/extraction/other commands across the fixture's repeated push/pop scopes. + +## Is rule-execution difference enough to measure encoding overhead? + +It is necessary, but not sufficient. + +After PR #61, the existing report can correctly expose: + +- generated maintenance rulesets such as `@rebuilding` and `@parent`; +- Search, Apply, Execution overhead, Merge, and Rebuild; +- changes in source-named rulesets after encoding. + +That is enough to identify Math's main bottleneck and Luminal's changed Search +shape. It is not enough for Pointer, eggcc, Hardboiled, or Herbie because it +still leaves these costs in one residual: + +- source versus generated typechecking; +- encoding generation and parsing of emitted text; +- generated desugaring; +- function/rule installation; +- ruleset assembly and per-invocation plan construction; +- top-level actions and command work. + +In particular, looking only at the five currently reported ruleset phases would +misclassify eggcc's approximately 253 ms generated-maintenance assembly cost as +frontend or generic outside overhead. + +## Minimal permanent reporting change + +A narrow reporting PR should add measurement before optimization PRs. + +1. Add `Assembly` before `Search` to each per-ruleset timing record. It should + include first-use cached-plan construction and rebuilding the executable + ruleset for each invocation. +2. Add disjoint outside-ruleset leaves under two explicit parents: + - Lowering / Parse, total Typecheck, and Other; + - Commands / Install, Actions/input, and Other/schedules; + - residual derived per observation from process wall time. +3. Charge source typechecking performed in the separate original-typechecker + e-graph to the outer total Typecheck leaf. Keeping one shared Typecheck leaf + makes the off-versus-encoded delta directly show the checking added by the + encoding, without storing mode-specific fields. +4. Render an additive per-file slowdown table like the one above, in addition + to detailed ruleset rows. Do not infer "frontend" from the wall-time + residual after collection. +5. Preserve same-binary comparisons and one-thread execution for additive + Search/Apply timing. + +The diagnostic patch used here added 196 lines across five Rust files. That is +too broad and ad hoc to retain as production code, but it validates the phase +boundaries for a smaller, reviewed implementation. A four-round clean-versus- +instrumented timer-tax comparison is retained at +`/tmp/term-overhead-timer-tax-off-v1.jsonl`; machine contention made its wall +CI inconclusive, while RSS was indistinguishable on nearly all workloads. A +production PR should include a cleaner timer-tax check. + +## Consequences for the unification roadmap + +The measurements argue against a single "replace encoded UF" campaign. + +- Preserve the one-relational-UF destination, but treat it as the Math-focused + and maintenance-focused track. +- Typed emission that removes generated parsing/desugaring/typechecking is the + direct track for Pointer and a large part of eggcc, Hardboiled, Luminal, and + Herbie. +- Identity/view fusion that restores narrow source query shapes is at least as + important as UF fusion for Luminal. +- Cache or eliminate repeated assembly of generated maintenance rulesets; + otherwise eggcc can spend more time preparing `@rebuilding` than executing + its Search/Apply/Merge phases. +- Keep top-level encoded action lowering visible. It is about 65 ms of + Luminal's slowdown even after the frontend is separated. + +The performance target should be evaluated per workload as well as in suite +aggregate. A change that fixes Math's `@rebuilding` cost can leave Pointer and +Luminal almost untouched, while a typed-frontend change can make Pointer much +faster and barely move Math. + +## P0b implementation evidence ledger + +Status: complete on `codex/term-encoding-always-on` from +`5ead0a0cacf847a129294a870de13503f2d7f9c4`. + +Smallest falsifiable contract: a synthetic V3 observation assigns distinct +nanosecond values to every exclusive leaf under Lowering, Commands, and +Ruleset. The report must display every leaf once, and the leaves plus the +derived residual must reconstruct external wall time. A real CLI fixture must +also show nonzero source Parse, Typecheck, Commands / Install, Commands / +Actions, and Ruleset / Assembly values. + +Current hypothesis: most of the previously unexplained proof/term-encoding +slowdown can be localized without splitting source and generated passes. Total +Typecheck is enough to answer how much extra typechecking the encoded mode +adds, provided typechecking done by the cloned source checker is charged to the +outer e-graph. Ruleset / Assembly must include first-use cached-plan creation +and per-invocation executable-ruleset materialization, while core execution +setup belongs to Ruleset / Execution overhead. + +Falsifiers: + +- nested command or schedule time appears both in Commands and Ruleset; +- source typechecking in encoded modes falls into Lowering / Other; +- generated parsing is omitted from Lowering / Parse; +- internal rebuild-rule assembly is recorded again outside Ruleset / Rebuild; +- the synthetic leaves plus residual do not equal wall time; +- timer instrumentation causes a material wall-time regression in an + instrumented-versus-clean same-treatment comparison. + +Evidence to retain: focused Rust producer/CLI tests, focused Python +schema-analysis-rendering tests, `make check`, `make benchmark-smoke`, the Rich +and Markdown width matrix, one real off-versus-proofs phase report, and a +timer-tax comparison. Failed hypotheses and inconclusive timing intervals stay +recorded rather than being discarded. + +Implementation result: + +- The producer and consumer contract is now V3. The initial tests failed on the + missing nested process schema, Assembly field, and analysis types, then pass + with all thirteen leaves reconstructing synthetic wall time exactly. +- `/tmp/term-overhead-off-proofs-v3.jsonl` contains four fresh rounds for all + six default workloads. The suite proof/off wall ratio is 3.04–3.16x. Its + phase tables leave small residual shares on the substantive workloads and + make the previously hidden Typecheck, Install, Actions/input, and Assembly + changes explicit. +- `/tmp/term-overhead-timer-tax-v3.jsonl` compares ten rounds of the + instrumented off mode with a temporary V3-compatible `5ead0a0` control. The + suite means are 1.7591 s versus 1.7408 s, a 1.0105x point ratio; the displayed + 95% interval rounds to 1.00–1.02x. This rejects a 5–10% suite-level timer tax, + though it does detect a small roughly 1% effect. +- `/tmp/term-overhead-timer-tax-proofs-v3.jsonl` repeats the control comparison + for proofs over four rounds; the suite interval is 0.873–1.07x and therefore + inconclusive, with every per-file wall interval including 1. +- Focused and six-file Rich reports render successfully at widths 80, 119, 120, + 160, and 200. Widths 80 and 119 emit exactly one detailed-report warning; + wider reports emit none. Markdown output is byte-identical across all five + widths for each scope. +- `make check` and `make benchmark-smoke` both pass in the implementation + worktree. + +## Flat mechanism-ledger follow-up + +Status: supersedes the fixed nested V3 transport above while retaining its +timer sites and the ruleset Assembly measurement. + +The persisted timing summary is now one sorted list of exclusive leaves: + +```json +{"schema_version":3,"timings":[{"path":["program","search","fusion_grow"],"ns":123}]} +``` + +The first path segment is the additive responsibility shown in the report; +deeper segments retain diagnostic resolution. The stable responsibilities are +Typecheck, Frontend, Program, Equality, and Commands. Residual remains derived +as process wall time minus the sum of every leaf. No parent total is stored. +Ruleset names are separate path segments, so names containing `/` cannot be +misparsed. + +Rulesets receive an explicit timing role when declared. Program rules write +Assembly, Search, Apply, Execution, and Merge under `program`; their native +Rebuild tail writes under `equality/rebuild`. Encoded maintenance rules write +all phases under `equality`. Thus the net cost of replacing native rebuilding +with relational maintenance is an ordinary candidate-minus-baseline Equality +difference, not a reporting-time credit calculation or an `@`-prefix guess. + +Checks have their own `commands/check` leaf. The transient backend query and +the surrounding compilation/validation overhead are charged there in both off +and encoded modes. The motivating claim that term-mode checks themselves were +showing up as the default ruleset was falsified by check-only CLI probes: the +old report recorded no transient check ruleset in either mode. Hardboiled has a +real default `(run)`. Keeping the explicit check leaf still removes the +ambiguity and prevents future routing asymmetry. + +One boundary remains intentionally command-scoped: a top-level action such as +`(union ...)` can trigger `flush_updates` and native rebuilding, but its +transient backend report is not a named ruleset run. That entire interval is +therefore recorded under `commands/actions`, not `equality/rebuild`. + +The fresh six-round report confirms the separation on real fixtures. +Hardboiled records `commands/check` means of 1.697 ms off and 3.299 ms term, +while its independent default-ruleset Search means are 60.349 ms and 84.870 +ms. Herbie records 0.175 ms and 0.241 ms for checks. Check evaluation is +therefore visible without being mistaken for transformed program-rule Search. + +At `--detail phases`, presentation starts with one additive +slowdown-decomposition table with a Suite row and one row per file. At +`--detail rulesets`, one driver panel per file unfolds exactly the Program and +Equality cells from that table. Program children contain source rules' own five +execution phases; Equality children contain every encoded-maintenance ruleset +and one global native-Rebuild replacement row when its delta is nonzero. The +two parent phase summaries retain the unique diagnostic question from the +removed global rollup: whether Program or Equality cost is Assembly, Search, +Rebuild, or another execution phase. Up to five source children plus an exact +per-group Other are shown, while the small fixed set of nonzero maintenance +children is shown in full. Native Rebuild is never attributed to whichever +source ruleset happened to trigger it. + +### Readability rationale + +The headline table follows a task-first rather than decorative color design. +A [controlled IEEE VIS table-reading +study](https://ieeexplore.ieeevis.org/year/2024/program/paper_v-full-1288.html) +found that visual aids are task dependent: color and bar encodings help some +extrema tasks, while row striping performs better for some complex comparison +tasks. [W3C table guidance](https://www.w3.org/WAI/tutorials/tables/tips/) +likewise recommends row orientation aids with sufficient contrast, and +[WCAG's use-of-color guidance](https://www.w3.org/WAI/WCAG20/Understanding/use-of-color) +requires that color not be the only signal. + +Accordingly, percent share comes first for vertical comparison, every report +table uses the same compact header-rule style, and `◆` identifies the largest +absolute mechanism share. Rich and interactive reports also bold that dominant +cell and dim contributions below 5%. Expected added overhead is neutral; green +is reserved for improvements, while yellow and red are reserved for measurement +warnings and errors. Signed values and the `◆` marker keep the meaning +independent of color. The contributor panels stay textual rather than adding in-cell bars: +the primary task is finding a dominant role, ruleset, and phase, and bars would +add another renderer-specific encoding to an already dense diagnostic. + +### Complexity audit and minimization + +The retained complexity falls into six distinct responsibilities. Keeping +them separate makes it possible to decide which parts are measurement +requirements and which are only report presentation. + +| Layer | Added responsibility | Why it remains | Simplification retained | +| --- | --- | --- | --- | +| Engine phase boundaries | Measure seven process leaves and six exclusive ruleset phases, including Assembly | Without these boundaries, Pointer frontend time and eggcc plan construction return to an undifferentiated residual | Static path slices and one duration map; no mode-specific timer structs | +| Semantic routing | Route source rules, relational equality maintenance, native Rebuild, and transient checks consistently | Equality is a responsibility implemented differently by the two treatments, so name-prefix inference or a report-time credit gives the wrong abstraction | One two-variant role enum; checks use one symmetric `commands/check` path | +| Scope-safe accounting | Preserve roles and accumulated time across push/pop, and subtract nested process/ruleset intervals from command timers | Otherwise nested schedules, checks, and rulesets are double-counted | One exclusive-subtraction boundary around commands and lowering; Residual verifies closure | +| Wire format | Persist exact measurements without fixing the set of diagnostic counters | A five-field record could answer today's headline but would lose the Assembly/Search evidence that chose different optimization PRs | One sorted open list of segmented `path -> ns` leaves; no parent totals and no separate per-ruleset record | +| Analysis | Align independent endpoint samples, derive Residual, and unfold Program and Equality into named children | Source own work, encoded maintenance, and native Rebuild must remain separate to keep every sign truthful | One generic exact-path sample map; the two parent groups equal the decomposition directly | +| Presentation and validation | Render one scan-first mechanism table and one compact driver panel per file; test additive closure at both hierarchy levels | Only the five buckets hides which ruleset carries Assembly/Search, while role totals falsely attach global Rebuild to source rules | Two mechanism parents, source top five plus exact Other, all maintenance children, and one native-Rebuild child | + +The minimization pass removed or avoided the main sources of accidental +complexity: + +- fixed nested timing structs and a second per-ruleset wire schema were + replaced by the one open path ledger; +- the native-Rebuild "credit" disappeared because both equality + implementations are recorded under `equality` before analysis; +- `@`-prefix classification disappeared in favor of declaration-time roles; +- mechanism and ruleset-driver reports now share one aligned sample map; +- the global depth-two rollup and ten-column ruleset tables became one compact + four-column panel per file whose two parent rows directly match Program and + Equality, with truthful per-group children and an exact source remainder; +- report invariants use ordinary exceptions, so `python -O` cannot remove + cache-safety checks; +- the visual treatment uses existing table primitives rather than adding bar + geometry or renderer-specific calculations. + +Two tempting reductions would make the design simpler only on paper. +Collapsing persistence to the five headline buckets would make the measured +eggcc Assembly and Luminal Search costs inseparable. Inferring maintenance +from generated names would remove the role enum but make semantics depend on a +printer convention. Neither is retained. + +The one plausible future reduction is to carry the semantic role inside the +engine's aggregated `RunReport`. That could remove the second role map used to +survive push/pop, but it would also make a benchmark-accounting concept part of +the public cross-crate report type. It is deferred until the role is useful to +the engine itself. Likewise, the extra timers can be gated if this moves +upstream and the measured tax becomes unacceptable; the current control run +does not justify that branch. + +As a review-surface count against `5ead0a0`, the current implementation is net +`+329` Rust source lines including the new 74-line timer module and inline +tests, net `+175` Python report lines, net `+345` external test/snapshot lines, +and net `+53` README lines. These are diff counts rather than runtime +complexity: the Rust report transport itself shrank while replacing V2, and +the snapshots contain no executable logic. The largest irreducible pieces are +the exclusive timing boundaries and their tests; the open-map transport and +mechanism/name projection are the parts deliberately kept small. + +The final driver redesign reduced production Python report complexity +from the pre-amendment net `+218` lines to `+175`: it deleted +`PhaseRollupView`, `_phase_rollups`, the global rollup renderer, endpoint-total +ruleset confidence intervals, the ten-column ruleset table, and the per-table +row-guide styling switch. External +validation grew because it now locks down direct mechanism-parent equality, +per-parent child additivity, deterministic phase threshold, and all report +renderers and supported widths. That growth is test-only; no second runtime +analysis or presentation path remains. + +### Fresh off-versus-term evidence + +The six-round report is: + +```text +/tmp/term-overhead-mechanisms-v3-20260812.jsonl +``` + +All 72 runs succeeded. The displayed suite wall ratio is `2.04–2.09x`. +The report treats endpoint samples as independent because the JSONL has no +persistent round-pair identity. Its additive suite slowdown is: + +| Mechanism | Delta | Share of slowdown | +| --- | ---: | ---: | +| Typecheck | +290 ms | 15.7% | +| Frontend/install | +270 ms | 14.6% | +| Program rules | +723 ms | 39.1% | +| Equality/rebuild | +416 ms | 22.5% | +| Commands | +116 ms | 6.28% | +| Residual | +34.0 ms | 1.84% | + +The file rows preserve the earlier diagnosis: Math is 62.5% Equality; +Pointer is 79.1% Typecheck plus Frontend; Luminal is 46.3% Program and only +4.89% Equality; eggcc and Herbie remain mixed. The small residual is the +accounting self-check that the table explains nearly all of the observed +slowdown. + +### Instrumentation-tax control + +The clean-control report is: + +```text +/tmp/term-overhead-timer-tax-leaves-v3-20260812.jsonl +``` + +It compares ten off-mode rounds of this instrumented build against a temporary +clean `5ead0a0` build. The clean build received only a compatibility adapter +that emits the same flat V3 leaf count and serialization shape with the new +process and Assembly values fixed at zero; it does not execute their timers. +This holds timing-summary serialization approximately constant while isolating +the additional timer sites. + +The clean suite mean was `1.723830 s`; the instrumented suite mean was +`1.727558 s`, for an instrumented/clean point ratio of `1.00216x`. The report's +95% interval is `0.995–1.01x` and includes 1. This run therefore detects no +suite-level timer slowdown and rules out a 5–10% tax under the measured +off-mode workload mix. diff --git a/term-encoding-unification.md b/term-encoding-unification.md new file mode 100644 index 00000000..cf2ff215 --- /dev/null +++ b/term-encoding-unification.md @@ -0,0 +1,477 @@ +# One execution path for equality and proofs + +- Status: feasibility brief, not an implementation plan yet +- Date: 2026-08-12 +- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` +- Paper/engineering companion: [`encoding-architecture-bridge.md`](encoding-architecture-bridge.md) +- Incremental implementation sequence: + [`incremental-unification-pr-roadmap.md`](incremental-unification-pr-roadmap.md) +- Current overhead decomposition: + [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) + +## Question + +Can egglog remove the independent normal and term/proof execution paths, make +one path universal, and keep the proof-disabled cost within roughly 5-10% of +today's normal mode? + +There are two materially different versions of that goal: + +1. **One proof-capable execution path, with evidence disabled by default.** + This looks feasible. Its disabled mode can be based on today's native + representation and should be held to a near-zero overhead gate. +2. **Always collect enough evidence to extract arbitrary proofs, within 5-10%.** + This is not supported by current measurements. It needs a separate, + deliberately optimistic lower-bound experiment before becoming a design + constraint. + +## Short answer + +Do not try to make the current generated term program the sole production +representation. On current `main`, term-only mode is **2.01-2.12x** normal wall +time across the six-workload suite and uses **1.59-2.26x** peak RSS. The gap is +not one removable proof feature: term-only already carries `Unit` rather than +real proofs. It comes from the physical representation and compiler pipeline: +extra term/view/UF tables, wider rows, rewritten queries, generated maintenance +rules and schedules, and a second desugar/typecheck pass. + +A single path is still a good complexity goal, but the likely destination is: + +```text +source + -> one parse/desugar/typecheck pipeline + -> typed e-graph operations + -> one equality kernel + - native/fused storage and rebuild + - NoEvidence or ProofEvidence sidecar + -> one rule engine +``` + +The current term encoding is valuable as an executable specification of +equality and proof semantics during migration. Once the typed kernel has parity, +its literal output should stop being a production execution mode. A printer +derived from the same typed encoding artifact can remain for the paper, +differential tests, and debugging without remaining an independently maintained +execution path. + +Calling that destination "term encoding" is reasonable if term encoding means +the semantic decomposition -- stable terms, interning, equivalence, congruence, +and justifications. It should not mean materializing that decomposition as +ordinary user-language tables and rules on the main backend. + +## What is duplicated today + +### Normal path + +```text +parse -> desugar -> typecheck -> remove globals -> typed commands + -> constructor/function tables + native Union actions + -> backend union-find + native rebuild +``` + +### Term/proof path + +```text +parse -> desugar -> typecheck in a cloned EGraph -> proof-normal-form checks + -> remove globals + -> generate new source AST: + term relations + FD views + @UF tables + indexes + rewritten bodies/actions + maintenance rules/schedules + -> desugar again -> typecheck again -> remove generated globals again + -> generic table/rule execution + -> (proof mode) retain the original typed program for checking +``` + +The second path is not a small option around the first. It independently models: + +- constructor creation and interning; +- explicit union and congruence; +- rule-body matching and rule-head construction; +- globals and top-level actions; +- custom functions and merge expressions; +- delete and subsume; +- path compression and rebuilding; +- container canonicalization; +- input loading and extraction; +- proof premises, proof nodes, extraction, simplification, and checking. + +The `egglog/src/proofs` directory currently has about 11,949 lines of +production Rust, excluding `proof_tests.rs`. Roughly 6,999 of those lines are in +the encoding-facing modules (`proof_encoding*`, `proof_head`, `proof_fresh`, and +`proof_container_rebuild`), in addition to an 877-line encoding design document. +Not all of those lines would disappear under a native proof path, but this shows +where the representational duplication lives; the checker, proof algebra, and +proof format are separate assets worth preserving. + +## Current measurements + +All benchmark comparisons used the same release binary at the baseline SHA, +four fresh rounds per endpoint/file, one thread, and a 120-second per-process +timeout. The exact cache is +`/tmp/term-encoding-always-on-bd4752e.jsonl`. + +### Execution and memory + +| Comparison | Suite wall-time ratio (95% CI) | Notable range | Peak-RSS range | +| --- | ---: | ---: | ---: | +| term vs off | 2.01-2.12x | 1.37-1.51x eggcc; 3.28-3.50x Luminal | 1.59-2.26x | +| proofs vs term | 1.48-1.59x | 1.26-1.37x eggcc; 1.70-1.78x Math | 1.37-1.78x | +| proofs vs off | 3.04-3.30x | 1.79-2.00x eggcc; 5.04-5.98x Luminal | 2.19-3.94x | + +`proofs` here means proof data generation for workloads ending in ordinary +`check`s. It does not include automatic proof extraction and verification. + +Per-workload term-only wall time: + +| Workload | Off | Term | Ratio (95% CI) | +| --- | ---: | ---: | ---: | +| Math | 357-395 ms | 817-835 ms | 2.09-2.32x | +| eggcc 2mm pass 1 | 813-885 ms | 1.20-1.25 s | 1.37-1.51x | +| Pointer small | 7.48-8.82 ms | 14.8-17.6 ms | 1.76-2.24x | +| Hardboiled conv1d | 113-121 ms | 218-227 ms | 1.83-1.98x | +| Luminal Llama | 369-391 ms | 1.27-1.30 s | 3.28-3.50x | +| Herbie | 53.7-55.2 ms | 107-111 ms | 1.95-2.05x | + +### Generated-program expansion + +`--mode desugar` exposes the extra compiler and representation work. + +| Workload | Normal output | Term output | Structural change | +| --- | ---: | ---: | --- | +| Math mini | 98 lines / 5,068 bytes | 521 / 29,974 | 24 -> 38 rules; 1 -> 6 schedules; 13 indexes added | +| eggcc 2mm pass 1 | 11,901 / 570,604 | 28,429 / 1,733,236 | 660 -> 1,203 rules; 2 -> 55 schedules; 417 indexes added | +| Luminal Llama | 7,058 / 515,895 | 69,607 / 4,274,991 | 491 -> 2,429 rules; 6 -> 29 schedules; 1,899 indexes added | + +For eggcc, 238 constructors plus 17 functions become 611 generated functions. +For Luminal, 117 constructors plus 1,646 functions become 3,548 functions; +the large static graph's nullary globals are a major contributor. + +Frontend-only `hyperfine` measurements (`--mode desugar`, two warmups, ten +runs, output discarded) were: + +| Workload | Normal mean | Term mean | Ratio | +| --- | ---: | ---: | ---: | +| eggcc 2mm pass 1 | 50.6 +/- 0.4 ms | 207.5 +/- 1.8 ms | 4.10x | +| Luminal Llama | 78.7 +/- 2.8 ms | 505.1 +/- 9.4 ms | 6.42x | + +### Runtime mechanisms + +The phase/ruleset data shows that deleting only the second frontend pass would +not reach the target: + +- On Math, generated `@rebuilding` costs 382-397 ms and `@parent` costs + 43.1-47.6 ms. Native rebuilding is faster even though its 135-179 ms appears + as an explicit cost that term mode reports as zero. +- On Luminal, transformed user-rule search rises from about 5.3-5.7 ms to + 475-477 ms. Generated maintenance is only a few milliseconds there; the + view-based query shape itself is the dominant runtime problem. +- On eggcc, the term frontend adds about 157 ms before execution, while the + full wall-time delta is roughly 0.37 s. Both compiler expansion and runtime + representation matter. + +### Language coverage + +The current support gate has 17 distinct unsupported-reason variants. The +checked-in unsupported snapshot contains 48 files out of 162 non-header, +non-`fail-typecheck` `.egg` corpus files. An always-on path cannot ship until +those are either supported by the common semantics or intentionally removed +from the language. + +Several restrictions are artifacts of the encoding rather than intentional +language semantics: function lookups in actions, tuple outputs, user-written +`begin`, merge action blocks, eq-sort `:no-merge`, user indexes, custom sorts, +and some primitive/container result shapes. A single native proof-capable path +should explain evidence for the underlying operation instead of rejecting the +surface syntax because a generated program cannot express it. + +### Experiment ledger + +| Hypothesis | Distinguishing prediction | Observation | Status | +| --- | --- | --- | --- | +| The second compiler pass explains most term overhead | Recorded runtime phases should be close to normal once outside-of-ruleset time is excluded | Math still spends about 434 ms in generated maintenance; Luminal search rises by about 471 ms | Rejected as a sufficient explanation | +| Generated maintenance is the dominant runtime cost | `@rebuilding` and `@parent` should explain most of every file's delta | True for much of Math, false for Luminal, where transformed user queries dominate | Workload-specific, not sufficient | +| Proof-node construction is the main reason term mode is slow | Term-only, with `Unit` proof columns, should be near normal | Term-only is 2.01-2.12x and 1.59-2.26x RSS | Rejected | +| A fused equality kernel can provide one path near normal cost | A no-evidence seam over native effects should benchmark within 1.05-1.10x | Not yet tested | Active; E1 is the next probe | + +Exact benchmark commands: + +```bash +./bench.py \ + --target . --compare-target . \ + --treatment term --compare-treatment off \ + --rounds 4 --timeout-sec 120 \ + --report /tmp/term-encoding-always-on-bd4752e.jsonl \ + --format markdown --detail rulesets + +./bench.py \ + --target . --compare-target . \ + --treatment proofs --compare-treatment term \ + --rounds 4 --timeout-sec 120 \ + --report /tmp/term-encoding-always-on-bd4752e.jsonl \ + --format markdown --detail phases +``` + +Representative frontend probe: + +```bash +hyperfine --warmup 2 --runs 10 --shell=zsh \ + 'target/release/egglog-experimental --mode desugar benchmarks/luminal-llama.egg >/dev/null 2>&1' \ + 'target/release/egglog-experimental --term-encoding --mode desugar benchmarks/luminal-llama.egg >/dev/null 2>&1' +``` + +## Why the current relational representation misses 5-10% + +The normal backend already implements the same semantic jobs in specialized +data structures: + +- one constructor/function table is both lookup structure and canonical view; +- one native union-find stores equivalence compactly; +- native rebuild uses occurrence information without running user-level rules; +- queries match the original, narrower rows; +- construction does not need a persistent term row, view row, and `Unit` proof + column for every application; +- schedules do not need maintenance spliced after user rulesets; +- source commands are not generated, parsed, and typechecked a second time. + +To bring the current term path near normal, all of those differences would +need to be fused away. At that point the physical implementation would be the +native equality kernel again, preferably behind a cleaner typed interface. + +Backend peepholes that recognize generated names such as `@UF_*` and +`@*View` would demonstrate a performance floor, but they are a poor final +architecture: they preserve the large compiler, couple the backend to generated +syntax, and create a hidden third execution path. + +## Recommended destination + +Use one typed semantic path with two evidence policies, not two programs. + +### 1. A typed equality kernel + +The frontend should lower every language construct once into a small set of +operations with explicit invariants, for example: + +- intern a constructor application and return its e-class; +- read or write a custom function row; +- union two e-classes with a cause; +- commit a batch and rebuild canonical columns; +- apply delete/subsume; +- run a typed rule firing with its substitution. + +The one production engine should implement these with today's fused tables, +union-find, and rebuild indexes. With the alternate backends being removed, +this interface should be chosen for clear semantics and useful compiler +staging, not as a lowest common denominator. `Backend::requires_term_encoding()` +should disappear with the backend split rather than be replaced by another +permanent execution-mode switch. + +### 2. Optional evidence attached to the same effects + +Each equality-producing effect should optionally return/store a compact receipt: + +- top-level or input fact (`Fiat`); +- rule firing and the matched row witnesses; +- explicit union/rewrite; +- constructor interning and congruence collision; +- custom-function merge result; +- rebuild/path-compression edge; +- container rebuild and normalization. + +The disabled policy should allocate nothing and avoid per-row dynamic dispatch. +The enabled policy should write compact IDs into a side arena, not ordinary +egglog relations. Proof expressions should be materialized root-first only when +requested. + +This makes proof availability a property of one runtime, while keeping the hot +representation specialized. + +### 3. Stable terms without a second e-graph + +Proofs need immutable syntactic identity even after rows are canonicalized, +deleted, or subsumed. Preserve that invariant in a compact `TermArena` or row +sidecar: + +```text +TermId -> constructor + child TermIds +row/eclass -> witness TermId +union edge -> CauseId +CauseId -> rule/merge/congruence receipt +``` + +This replaces the persistent term relations and proof-node relations without +losing the information the checker needs. + +### 4. A rule catalog instead of `proof_check_program` + +The checker needs normalized rule definitions, merge definitions, global facts, +and primitive validators. Store those once in an immutable typed `RuleCatalog` +shared by execution and checking. Do not retain a second full command stream +whose shape must stay synchronized with the encoded one. + +### 5. Keep proof semantics, remove encoding mechanics + +Likely keep and adapt: + +- the proof algebra and proof term format; +- `ProofStore`, simplification, and the independent checker; +- deterministic extraction policy; +- immutable term identity and typed primitive validators; +- proof snapshot tests. + +Likely delete or replace: + +- `ProofInstrumentor::add_term_encoding` and command-by-command AST rewriting; +- the cloned `original_typechecking` `EGraph` and second typecheck pass; +- generated term/view/`@UF` tables and `Unit` proof columns; +- generated occurrence-index declarations and maintenance schedules; +- generated path-compression, rebuild, cleanup, and subsume rules; +- proof nodes represented as normal e-graph function rows; +- support rejections caused only by the generated representation; +- `proof_check_program` as a duplicate program; +- the production `--term-encoding` execution mode after migration. + +### Likely implementation seams + +Current source already concentrates several equality effects at useful +boundaries: + +- `EGraph::resolve_command` in `egglog/src/lib.rs` is the frontend split that + should collapse back to one typed pipeline. +- `EGraph::declare_function` chooses constructor `MergeFn::UnionId`; this is + where a common constructor/interner contract can replace proof-specific view + declarations. +- `UnionAction::union` in `egglog/egglog-bridge/src/lib.rs` is the direct native + union write. +- `EGraph::rebuild` in the bridge owns container-first canonicalization and + table rebuild; it needs to report congruence/rebuild causes through the same + optional evidence policy. +- `InPlaceActionBuffer::push_bindings` and its scoped counterpart in + `core-relations/src/free_join/execute.rs` are where a successful rule match + becomes an action batch. + +The last item is probably the hardest design boundary. Native joins currently +need variable values to execute a head; proof reconstruction also needs stable +identities for the body rows that witnessed the match. Widening every binding +with row provenance would damage the disabled hot path. E3 therefore needs to +test a representation that is absent under `NoEvidence` and carries compact row +or receipt identities only under `ProofEvidence`. + +## How incremental desugaring fits + +Term encoding is a useful semantic decomposition of the language, but its +pieces should lower into typed internal operators, not recursively back into +egglog source. + +The migration can therefore be incremental: + +1. Normalize globals, nested expressions, and rule heads once into common typed + IR. +2. Give construction/interning one operator and route both normal and proof + behavior through it. +3. Give union, congruence, custom merge, and rebuild explicit cause-bearing + operators. +4. Move input, containers, delete/subsume, and extraction onto those operators. +5. Add the proof evidence policy and reconstruct the current proof format from + receipts. +6. Retain literal encoded output as a parity oracle while each family moves, + generated from the same typed artifact that feeds the fused lowerer. +7. Delete the old production mode once coverage, proof validity, and + performance gates pass; retain the derived printer only as a test/paper + asset if it remains useful. + +This is a strangler migration around semantic operations, not a flag-day +rewrite and not permanent coexistence of two execution semantics. + +## Options + +| Option | Complexity outcome | Performance outlook | Main risk | +| --- | --- | --- | --- | +| Make today's generated term program universal | Deletes native UF/rebuild, retains the large encoder | Poor without fusing away its defining representation | More compiler/backend pattern coupling; incomplete language | +| Native single path plus optional proof sidecar | Deletes the source encoder and support split | Disabled mode can be close to current normal; enabled cost unknown | Capturing sound merge/rebuild/rule causes in the native engine | +| Typed encoding IR plus one fused physical lowering | One language semantics, one engine, and a derived reference printer | Fused lowering can retain current native speed | Designing a stable semantic/fusion boundary without building another framework | +| Keep both paths but isolate/shared utilities | Smaller near-term refactor | No forced regression | Does not remove semantic duplication or support drift | + +The recommendation is the second and third options together: a common typed +encoding IR, one fused native kernel, an optional proof-evidence policy, and a +reference printer derived from that same IR. The companion architecture note +explains how slotted then proof encoding can compose at this boundary. + +## Falsifying experiment ladder + +Large production edits should wait until these floors are measured in order. + +### E0: frozen reference matrix + +Keep the current off/term/proofs measurements and add exact output parity for +the six benchmark files. This is the immutable comparison set. + +### E1: `NoEvidence` seam + +Route native construction, union, merge, and rebuild through the proposed +evidence interface, with a zero-sized disabled implementation. Record nothing. + +Gate: + +- no semantic or snapshot delta; +- <=1.05x suite wall time and <=1.10x on every file; +- <=1.05x peak RSS; +- no per-row allocation and no dynamic dispatch in the hot loop. + +If this fails, the interface boundary is wrong before proof design begins. + +### E2: immutable-term floor + +Record only the stable `TermId`/witness arena needed by any native proof design. +Do not record union causes or build proof nodes. + +This isolates the irreducible cost of keeping syntactic identity. If it already +exceeds 1.10x, reuse existing row IDs more aggressively or abandon an +always-recording 5-10% target. + +### E3: receipt-only floor + +Record the smallest sound cause for native rule firings, unions, congruence, +merge, and rebuild. Do not extract, simplify, or verify a proof. + +This is the decisive optimistic lower bound for "proofs always available at +5-10%." If it misses the gate, selector or extractor work cannot rescue the +capture cost. + +### E4: one end-to-end witness + +On a tiny fixture containing construction, a rewrite, congruence, and a custom +merge, reconstruct the existing proof format from receipts and validate it with +the independent checker. Compare exact propositions, not necessarily exact +pretty-print shape. + +### E5: semantic expansion + +Add containers, globals/scopes, input, delete/subsume, action lookups, tuple +outputs, and user indexes one family at a time. Every accepted family must flip +its current unsupported canary while preserving the existing normal corpus. + +### E6: deletion gate + +Delete the old source encoding only after: + +- every non-failing corpus file uses the common path; +- all explicit proof fixtures validate; +- the six-file disabled-evidence suite stays within the agreed wall/RSS gate; +- proof-enabled overhead is reported separately from disabled overhead; +- the printed reference encoding, if retained, is not callable as a separate + production execution mode. + +## Decision + +The current source-to-source term encoding cannot plausibly be tuned from +2.01-2.12x to 1.05-1.10x by deleting a few proof features. Reaching that band +requires removing the generated physical representation: duplicate tables, +query expansion, maintenance rules, schedule injection, and the second compiler +pass. + +One execution path is nevertheless plausible and likely the best way to reduce +repo complexity. Build it from the native fast path, make equality/provenance +explicit in a typed encoding IR, and make evidence an optional sidecar. Use the +literal term encoding as the semantic oracle during migration, then delete its +production path while retaining a derived reference printer if the paper and +tests still need it. diff --git a/tests/__snapshots__/test_report_rendering.ambr b/tests/__snapshots__/test_report_rendering.ambr index ba3164ba..89347f2d 100644 --- a/tests/__snapshots__/test_report_rendering.ambr +++ b/tests/__snapshots__/test_report_rendering.ambr @@ -42,223 +42,169 @@ | math.egg | 89.8–101.9 MiB | 85.0–97.1 MiB | 0.867–1.04x | CI includes 1 | | rewrite.egg | 118.4–130.5 MiB | 137.5–149.6 MiB | 1.08–1.23x | higher RSS | - ## Phase comparison + ## Slowdown decomposition - *Endpoint cells show a 95% CI (or one-round point) and that phase's share of endpoint wall time. Delta is the signed candidate − baseline mean; Δ contribution is the phase's share of the wall-time change and may be negative or exceed 100% when phases offset. Execution overhead is stored per-ruleset unattributed time. Outside recorded rulesets is wall time minus all five recorded phases; ! marks a negative residual.* + | File | Wall Δ | Typecheck | Frontend | Program | Equality | Commands | Residual | + | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | + | Suite total | +200 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% +116 ms | 0% 0 ms | 0% 0 ms | +42.0% +84.0 ms | + | math.egg | -200 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% -116 ms | 0% 0 ms | 0% 0 ms | +42.0% -84.0 ms | + | rewrite.egg | +400 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% +232 ms | 0% 0 ms | 0% 0 ms | +42.0% +168 ms | - ### Phase comparison — math.egg + *Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* - | Phase | Baseline (95% CI · wall) | Candidate (95% CI · wall) | Delta | Δ contribution | - | --- | ---: | ---: | ---: | ---: | - | Search | 353–455 ms · 40.0% | 273–375 ms · 40.0% | -80.0 ms | +40.0% | - | Apply | 159–205 ms · 18.0% | 123–169 ms · 18.0% | -36.0 ms | +18.0% | - | Execution overhead | 0–0 ns · 0% | 0–0 ns · 0% | 0 ns | 0% | - | Merge | 100–100 ms · 9.90% | 100–100 ms · 12.3% | 0 ns | 0% | - | Rebuild | 40.0–40.0 ms · 3.96% | 40.0–40.0 ms · 4.94% | 0 ns | 0% | - | Outside recorded rulesets | 231–338 ms · 28.1% | 147–254 ms · 24.7% | -84.0 ms | +42.0% | + ## Ruleset drivers - ### Phase comparison — rewrite.egg + *Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those cells and alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, and Merge; Equality children contain every encoded maintenance ruleset plus one global Native rebuild replaced row. ↳ marks children in every format. Zero children are hidden. Source children are ranked by absolute own-work Δ (top 5 plus an exact per-group Other); every nonzero maintenance child is shown. Important phases include every \|phase Δ\| ≥ max(1 ms, 10% of \|row Δ\|), always include the dominant phase (◆), and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases.* - | Phase | Baseline (95% CI · wall) | Candidate (95% CI · wall) | Delta | Δ contribution | - | --- | ---: | ---: | ---: | ---: | - | Search | 753–855 ms · 40.0% | 0.913–1.01 s · 40.0% | +160 ms | +40.0% | - | Apply | 339–385 ms · 18.0% | 411–457 ms · 18.0% | +72.0 ms | +18.0% | - | Execution overhead | 0–0 ns · 0% | 0–0 ns · 0% | 0 ns | 0% | - | Merge | 100–100 ms · 4.98% | 100–100 ms · 4.15% | 0 ns | 0% | - | Rebuild | 40.0–40.0 ms · 1.99% | 40.0–40.0 ms · 1.66% | 0 ns | 0% | - | Outside recorded rulesets | 651–758 ms · 35.0% | 819–926 ms · 36.2% | +168 ms | +42.0% | + ### Ruleset drivers — math.egg - ## Ruleset comparison + | Driver | Δ | Wall share | Important phase changes | + | --- | ---: | ---: | --- | + | Program rules — own work | -116 ms | +58.0% | ◆ Search -80.0 ms; Apply -36.0 ms | + | ↳ simplify | -84.0 ms | | ◆ Search -60.0 ms; Apply -24.0 ms | + | ↳ finish | -32.0 ms | | ◆ Search -20.0 ms; Apply -12.0 ms | + | Equality/rebuild — net | 0 ns | 0% | 0 ns | - *Totals show a 95% CI or one-round point. S/A/Exec/M/R are signed candidate − baseline mean deltas for Search, Apply, Execution overhead (stored unattributed time), Merge, and Rebuild.* + *Program + Equality account for +58.0% of this file's wall-time change. Source rules shown: 2/2. Maintenance rules shown: none.* - ### Ruleset comparison — math.egg + ### Ruleset drivers — rewrite.egg - | Ruleset | Baseline total | Candidate total | Total Δ | S Δ | A Δ | Exec Δ | M Δ | R Δ | - | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | - | simplify | 481–588 ms | 397–504 ms | -84.0 ms | -60.0 ms | -24.0 ms | 0 ns | 0 ns | 0 ns | - | finish | 171–212 ms | 139–180 ms | -32.0 ms | -20.0 ms | -12.0 ms | 0 ns | 0 ns | 0 ns | + | Driver | Δ | Wall share | Important phase changes | + | --- | ---: | ---: | --- | + | Program rules — own work | +232 ms | +58.0% | ◆ Search +160 ms; Apply +72.0 ms | + | ↳ simplify | +168 ms | | ◆ Search +120 ms; Apply +48.0 ms | + | ↳ finish | +64.0 ms | | ◆ Search +40.0 ms; Apply +24.0 ms | + | Equality/rebuild — net | 0 ns | 0% | 0 ns | - ### Ruleset comparison — rewrite.egg - - | Ruleset | Baseline total | Candidate total | Total Δ | S Δ | A Δ | Exec Δ | M Δ | R Δ | - | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | - | simplify | 0.901–1.01 s | 1.07–1.18 s | +168 ms | +120 ms | +48.0 ms | 0 ns | 0 ns | 0 ns | - | finish | 331–372 ms | 395–436 ms | +64.0 ms | +40.0 ms | +24.0 ms | 0 ns | 0 ns | 0 ns | + *Program + Equality account for +58.0% of this file's wall-time change. Source rules shown: 2/2. Maintenance rules shown: none.* ''' # --- # name: test_realistic_six_file_rich_120_snapshot ''' - ────────────────────────────────────────────────── Ruleset comparison ────────────────────────────────────────────────── - Totals show a 95% CI or one-round point. S/A/Exec/M/R are signed candidate − baseline mean deltas for Search, Apply, - Execution overhead (stored unattributed time), Merge, and Rebuild. - Ruleset comparison — math.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 41.9–53.9 ms 37.7–48.5 ms -4.79 ms -2.42 ms -1.21 ms -242 us -606 us -303 us - ruleset-10 38.4–49.4 ms 34.5–44.5 ms -4.39 ms -2.22 ms -1.11 ms -222 us -556 us -278 us - ruleset-09 34.9–44.9 ms 31.4–40.4 ms -3.99 ms -2.02 ms -1.01 ms -202 us -505 us -252 us - ruleset-08 31.4–40.4 ms 28.2–36.4 ms -3.59 ms -1.82 ms -909 us -182 us -454 us -227 us - ruleset-07 27.9–35.9 ms 25.1–32.3 ms -3.19 ms -1.62 ms -808 us -162 us -404 us -202 us - ruleset-06 24.4–31.4 ms 22.0–28.3 ms -2.79 ms -1.41 ms -707 us -141 us -354 us -177 us - ruleset-05 20.9–26.9 ms 18.8–24.3 ms -2.39 ms -1.21 ms -606 us -121 us -303 us -152 us - ruleset-04 17.4–22.5 ms 15.7–20.2 ms -1.99 ms -1.01 ms -505 us -101 us -252 us -126 us - ruleset-03 14.0–18.0 ms 12.6–16.2 ms -1.60 ms -808 us -404 us -80.8 us -202 us -101 us - ruleset-02 10.5–13.5 ms 9.42–12.1 ms -1.20 ms -606 us -303 us -60.6 us -152 us -75.8 us - - Showing 10 of 12 changed rulesets by absolute total delta. - Ruleset comparison — eggcc-extract.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 73.6–94.8 ms 69.2–89.1 ms -5.05 ms -2.91 ms -1.45 ms -145 us -364 us -182 us - ruleset-10 67.5–86.9 ms 63.5–81.7 ms -4.63 ms -2.67 ms -1.33 ms -133 us -333 us -167 us - ruleset-09 61.4–79.0 ms 57.7–74.3 ms -4.21 ms -2.42 ms -1.21 ms -121 us -303 us -152 us - ruleset-08 55.2–71.1 ms 51.9–66.9 ms -3.79 ms -2.18 ms -1.09 ms -109 us -273 us -136 us - ruleset-07 49.1–63.2 ms 46.1–59.4 ms -3.37 ms -1.94 ms -970 us -97.0 us -242 us -121 us - ruleset-06 43.0–55.3 ms 40.4–52.0 ms -2.95 ms -1.70 ms -848 us -84.8 us -212 us -106 us - ruleset-05 36.8–47.4 ms 34.6–44.6 ms -2.53 ms -1.45 ms -727 us -72.7 us -182 us -90.9 us - ruleset-04 30.7–39.5 ms 28.8–37.1 ms -2.11 ms -1.21 ms -606 us -60.6 us -152 us -75.8 us - ruleset-03 24.5–31.6 ms 23.1–29.7 ms -1.68 ms -970 us -485 us -48.5 us -121 us -60.6 us - ruleset-02 18.4–23.7 ms 17.3–22.3 ms -1.26 ms -727 us -364 us -36.4 us -90.9 us -45.5 us - - Showing 10 of 12 changed rulesets by absolute total delta. - Ruleset comparison — pointer-analysis-small.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 105–136 ms 103–133 ms -2.41 ms -1.45 ms -727 us -48.5 us -121 us -60.6 us - ruleset-10 96.6–124 ms 94.7–122 ms -2.21 ms -1.33 ms -667 us -44.4 us -111 us -55.5 us - ruleset-09 87.9–113 ms 86.1–111 ms -2.01 ms -1.21 ms -606 us -40.4 us -101 us -50.5 us - ruleset-08 79.1–102 ms 77.5–99.8 ms -1.81 ms -1.09 ms -545 us -36.4 us -90.9 us -45.5 us - ruleset-07 70.3–90.5 ms 68.9–88.7 ms -1.61 ms -970 us -485 us -32.3 us -80.8 us -40.4 us - ruleset-06 61.5–79.2 ms 60.3–77.6 ms -1.41 ms -848 us -424 us -28.3 us -70.7 us -35.4 us - ruleset-05 52.7–67.9 ms 51.7–66.5 ms -1.21 ms -727 us -364 us -24.2 us -60.6 us -30.3 us - ruleset-04 43.9–56.6 ms 43.0–55.4 ms -1.00 ms -606 us -303 us -20.2 us -50.5 us -25.2 us - ruleset-03 35.1–45.3 ms 34.4–44.3 ms -804 us -485 us -242 us -16.2 us -40.4 us -20.2 us - ruleset-02 26.4–33.9 ms 25.8–33.3 ms -603 us -364 us -182 us -12.1 us -30.3 us -15.2 us - - Showing 10 of 12 changed rulesets by absolute total delta. - Ruleset comparison — hardboiled.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 137–177 ms 140–180 ms +3.14 ms +1.94 ms +970 us +48.5 us +121 us +60.6 us - ruleset-10 126–162 ms 128–165 ms +2.88 ms +1.78 ms +889 us +44.4 us +111 us +55.5 us - ruleset-09 114–147 ms 117–150 ms +2.62 ms +1.62 ms +808 us +40.4 us +101 us +50.5 us - ruleset-08 103–133 ms 105–135 ms +2.35 ms +1.45 ms +727 us +36.4 us +90.9 us +45.5 us - ruleset-07 91.5–118 ms 93.3–120 ms +2.09 ms +1.29 ms +646 us +32.3 us +80.8 us +40.4 us - ruleset-06 80.0–103 ms 81.6–105 ms +1.83 ms +1.13 ms +566 us +28.3 us +70.7 us +35.4 us - ruleset-05 68.6–88.3 ms 70.0–90.1 ms +1.57 ms +970 us +485 us +24.2 us +60.6 us +30.3 us - ruleset-04 57.2–73.6 ms 58.3–75.1 ms +1.31 ms +808 us +404 us +20.2 us +50.5 us +25.2 us - ruleset-03 45.7–58.9 ms 46.7–60.1 ms +1.05 ms +646 us +323 us +16.2 us +40.4 us +20.2 us - ruleset-02 34.3–44.2 ms 35.0–45.1 ms +785 us +485 us +242 us +12.1 us +30.3 us +15.2 us - - Showing 10 of 12 changed rulesets by absolute total delta. - Ruleset comparison — luminal.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 169–218 ms 179–231 ms +11.6 ms +7.27 ms +3.64 ms +145 us +364 us +182 us - ruleset-10 155–199 ms 164–211 ms +10.6 ms +6.67 ms +3.33 ms +133 us +333 us +167 us - ruleset-09 141–181 ms 149–192 ms +9.67 ms +6.06 ms +3.03 ms +121 us +303 us +152 us - ruleset-08 127–163 ms 134–173 ms +8.70 ms +5.45 ms +2.73 ms +109 us +273 us +136 us - ruleset-07 113–145 ms 119–154 ms +7.73 ms +4.85 ms +2.42 ms +97.0 us +242 us +121 us - ruleset-06 98.6–127 ms 104–135 ms +6.77 ms +4.24 ms +2.12 ms +84.8 us +212 us +106 us - ruleset-05 84.5–109 ms 89.6–115 ms +5.80 ms +3.64 ms +1.82 ms +72.7 us +182 us +90.9 us - ruleset-04 70.4–90.7 ms 74.6–96.1 ms +4.83 ms +3.03 ms +1.51 ms +60.6 us +152 us +75.8 us - ruleset-03 56.3–72.5 ms 59.7–76.9 ms +3.87 ms +2.42 ms +1.21 ms +48.5 us +121 us +60.6 us - ruleset-02 42.2–54.4 ms 44.8–57.7 ms +2.90 ms +1.82 ms +909 us +36.4 us +90.9 us +45.5 us - - Showing 10 of 12 changed rulesets by absolute total delta. - Ruleset comparison — herbie.egg - - Ruleset Baseline total Candidate total Total Δ S Δ A Δ Exec Δ M Δ R Δ - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ruleset-11 201–259 ms 221–284 ms +23.0 ms +14.5 ms +7.27 ms +242 us +606 us +303 us - ruleset-10 184–237 ms 202–261 ms +21.1 ms +13.3 ms +6.67 ms +222 us +556 us +278 us - ruleset-09 167–215 ms 184–237 ms +19.1 ms +12.1 ms +6.06 ms +202 us +505 us +252 us - ruleset-08 151–194 ms 166–213 ms +17.2 ms +10.9 ms +5.45 ms +182 us +454 us +227 us - ruleset-07 134–172 ms 147–190 ms +15.3 ms +9.70 ms +4.85 ms +162 us +404 us +202 us - ruleset-06 117–151 ms 129–166 ms +13.4 ms +8.48 ms +4.24 ms +141 us +354 us +177 us - ruleset-05 100–129 ms 110–142 ms +11.5 ms +7.27 ms +3.64 ms +121 us +303 us +152 us - ruleset-04 83.7–108 ms 92.0–119 ms +9.57 ms +6.06 ms +3.03 ms +101 us +252 us +126 us - ruleset-03 66.9–86.2 ms 73.6–94.8 ms +7.66 ms +4.85 ms +2.42 ms +80.8 us +202 us +101 us - ruleset-02 50.2–64.6 ms 55.2–71.1 ms +5.74 ms +3.64 ms +1.82 ms +60.6 us +152 us +75.8 us - - Showing 10 of 12 changed rulesets by absolute total delta. - ─────────────────────────────────────────────────── Phase comparison ─────────────────────────────────────────────────── - Endpoint cells show a 95% CI (or one-round point) and that phase's share of endpoint wall time. Delta is the signed - candidate − baseline mean; Δ contribution is the phase's share of the wall-time change and may be negative or exceed - 100% when phases offset. Execution overhead is stored per-ruleset unattributed time. Outside recorded rulesets is wall - time minus all five recorded phases; ! marks a negative residual. - Phase comparison — math.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 138–177 ms · 15.7% 124–160 ms · 17.6% -15.8 ms +7.88% - Apply 68.9–88.7 ms · 7.84% 62.0–79.8 ms · 8.81% -7.88 ms +3.94% - Execution overhead 13.8–17.7 ms · 1.57% 12.4–16.0 ms · 1.76% -1.58 ms +0.788% - Merge 34.4–44.3 ms · 3.92% 31.0–39.9 ms · 4.40% -3.94 ms +1.97% - Rebuild 17.2–22.2 ms · 1.96% 15.5–20.0 ms · 2.20% -1.97 ms +0.985% - Outside recorded rulesets 669–718 ms · 69.0% 497–553 ms · 65.2% -169 ms +84.4% - - Phase comparison — eggcc-extract.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 275–355 ms · 20.9% 259–333 ms · 21.9% -18.9 ms +12.6% - Apply 138–177 ms · 10.5% 129–167 ms · 10.9% -9.45 ms +6.30% - Execution overhead 13.8–17.7 ms · 1.05% 12.9–16.7 ms · 1.09% -945 us +0.630% - Merge 34.4–44.3 ms · 2.62% 32.4–41.7 ms · 2.73% -2.36 ms +1.58% - Rebuild 17.2–22.2 ms · 1.31% 16.2–20.8 ms · 1.37% -1.18 ms +0.788% - Outside recorded rulesets 952–963 ms · 63.6% 839–842 ms · 62.0% -117 ms +78.1% - - Phase comparison — pointer-analysis-small.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 413–532 ms · 23.6% 405–522 ms · 23.1% -9.45 ms — - Apply 207–266 ms · 11.8% 202–261 ms · 11.6% -4.73 ms — - Execution overhead 13.8–17.7 ms · 0.786% 13.5–17.4 ms · 0.770% -315 us — - Merge 34.4–44.3 ms · 1.96% 33.7–43.5 ms · 1.93% -788 us — - Rebuild 17.2–22.2 ms · 0.982% 16.9–21.7 ms · 0.963% -394 us — - Outside recorded rulesets 1.19–1.26 s · 60.9% 1.20–1.27 s · 61.7% +15.7 ms — - - Phase comparison — hardboiled.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 551–710 ms · 25.2% 562–724 ms · 23.3% +12.6 ms +5.04% - Apply 275–355 ms · 12.6% 281–362 ms · 11.7% +6.30 ms +2.52% - Execution overhead 13.8–17.7 ms · 0.629% 14.0–18.1 ms · 0.583% +315 us +0.126% - Merge 34.4–44.3 ms · 1.57% 35.1–45.2 ms · 1.46% +788 us +0.315% - Rebuild 17.2–22.2 ms · 0.786% 17.6–22.6 ms · 0.729% +394 us +0.158% - Outside recorded rulesets 1.42–1.55 s · 59.3% 1.65–1.78 s · 62.2% +230 ms +91.8% - - Phase comparison — luminal.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 689–887 ms · 26.2% 730–940 ms · 23.2% +47.3 ms +7.88% - Apply 344–443 ms · 13.1% 365–470 ms · 11.6% +23.6 ms +3.94% - Execution overhead 13.8–17.7 ms · 0.524% 14.6–18.8 ms · 0.463% +945 us +0.158% - Merge 34.4–44.3 ms · 1.31% 36.5–47.0 ms · 1.16% +2.36 ms +0.394% - Rebuild 17.2–22.2 ms · 0.655% 18.3–23.5 ms · 0.579% +1.18 ms +0.197% - Outside recorded rulesets 1.65–1.84 s · 58.2% 2.17–2.38 s · 63.1% +525 ms +87.4% - - Phase comparison — herbie.egg - - Phase Baseline (95% CI · wall) Candidate (95% CI · wall) Delta Δ contribution - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Search 0.826–1.06 s · 27.0% 0.909–1.17 s · 22.8% +94.5 ms +9.00% - Apply 413–532 ms · 13.5% 455–585 ms · 11.4% +47.3 ms +4.50% - Execution overhead 13.8–17.7 ms · 0.450% 15.2–19.5 ms · 0.380% +1.58 ms +0.150% - Merge 34.4–44.3 ms · 1.12% 37.9–48.8 ms · 0.951% +3.94 ms +0.375% - Rebuild 17.2–22.2 ms · 0.562% 18.9–24.4 ms · 0.476% +1.97 ms +0.188% - Outside recorded rulesets 1.89–2.14 s · 57.4% 2.77–3.06 s · 63.9% +901 ms +85.8% - + ─────────────────────────────────────────────────── Ruleset drivers ──────────────────────────────────────────────────── + Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those cells and + alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, and Merge; Equality + children contain every encoded maintenance ruleset plus one global Native rebuild replaced row. ↳ marks children in + every format. Zero children are hidden. Source children are ranked by absolute own-work Δ (top 5 plus an exact per-group + Other); every nonzero maintenance child is shown. Important phases include every |phase Δ| ≥ max(1 ms, 10% of |row Δ|), + always include the dominant phase (◆), and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks + omitted nonzero phases. + Ruleset drivers — math.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work -29.1 ms +14.6% ◆ Search -15.8 ms; Apply -7.88 ms; Merge -3.94 ms; … + ↳ ruleset-11 -4.48 ms ◆ Search -2.42 ms; Apply -1.21 ms; … + ↳ ruleset-10 -4.11 ms ◆ Search -2.22 ms; Apply -1.11 ms; … + ↳ ruleset-09 -3.74 ms ◆ Search -2.02 ms; Apply -1.01 ms; … + ↳ ruleset-08 -3.36 ms ◆ Search -1.82 ms; … + ↳ ruleset-07 -2.99 ms ◆ Search -1.62 ms; … + ↳ Other (7 more source rulesets) -10.5 ms ◆ Search -5.66 ms; Apply -2.83 ms; Merge -1.41 ms; … + Equality/rebuild — net -1.97 ms +0.985% ◆ Rebuild -1.97 ms + ↳ Native rebuild replaced -1.97 ms ◆ Rebuild -1.97 ms + + Program + Equality account for +15.6% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + Ruleset drivers — eggcc-extract.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work -31.7 ms +21.1% ◆ Search -18.9 ms; Apply -9.45 ms; … + ↳ ruleset-11 -4.87 ms ◆ Search -2.91 ms; Apply -1.45 ms; … + ↳ ruleset-10 -4.47 ms ◆ Search -2.67 ms; Apply -1.33 ms; … + ↳ ruleset-09 -4.06 ms ◆ Search -2.42 ms; Apply -1.21 ms; … + ↳ ruleset-08 -3.65 ms ◆ Search -2.18 ms; Apply -1.09 ms; … + ↳ ruleset-07 -3.25 ms ◆ Search -1.94 ms; … + ↳ Other (7 more source rulesets) -11.4 ms ◆ Search -6.79 ms; Apply -3.39 ms; … + Equality/rebuild — net -1.18 ms +0.788% ◆ Rebuild -1.18 ms + ↳ Native rebuild replaced -1.18 ms ◆ Rebuild -1.18 ms + + Program + Equality account for +21.9% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + Ruleset drivers — pointer-analysis-small.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work -15.3 ms — ◆ Search -9.45 ms; Apply -4.73 ms; … + ↳ ruleset-11 -2.35 ms ◆ Search -1.45 ms; … + ↳ ruleset-10 -2.16 ms ◆ Search -1.33 ms; … + ↳ ruleset-09 -1.96 ms ◆ Search -1.21 ms; … + ↳ ruleset-08 -1.76 ms ◆ Search -1.09 ms; … + ↳ ruleset-07 -1.57 ms ◆ Search -970 us; … + ↳ Other (7 more source rulesets) -5.49 ms ◆ Search -3.39 ms; Apply -1.70 ms; … + Equality/rebuild — net -394 us — ◆ Rebuild -394 us + ↳ Native rebuild replaced -394 us ◆ Rebuild -394 us + + Program + Equality coverage is unavailable because wall time did not change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + Ruleset drivers — hardboiled.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work +20.0 ms +8.00% ◆ Search +12.6 ms; Apply +6.30 ms; … + ↳ ruleset-11 +3.08 ms ◆ Search +1.94 ms; … + ↳ ruleset-10 +2.82 ms ◆ Search +1.78 ms; … + ↳ ruleset-09 +2.57 ms ◆ Search +1.62 ms; … + ↳ ruleset-08 +2.31 ms ◆ Search +1.45 ms; … + ↳ ruleset-07 +2.05 ms ◆ Search +1.29 ms; … + ↳ Other (7 more source rulesets) +7.18 ms ◆ Search +4.52 ms; Apply +2.26 ms; … + Equality/rebuild — net +394 us +0.158% ◆ Rebuild +394 us + ↳ Native rebuild replaced +394 us ◆ Rebuild +394 us + + Program + Equality account for +8.16% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + Ruleset drivers — luminal.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work +74.2 ms +12.4% ◆ Search +47.3 ms; Apply +23.6 ms; … + ↳ ruleset-11 +11.4 ms ◆ Search +7.27 ms; Apply +3.64 ms; … + ↳ ruleset-10 +10.5 ms ◆ Search +6.67 ms; Apply +3.33 ms; … + ↳ ruleset-09 +9.51 ms ◆ Search +6.06 ms; Apply +3.03 ms; … + ↳ ruleset-08 +8.56 ms ◆ Search +5.45 ms; Apply +2.73 ms; … + ↳ ruleset-07 +7.61 ms ◆ Search +4.85 ms; Apply +2.42 ms; … + ↳ Other (7 more source rulesets) +26.6 ms ◆ Search +17.0 ms; Apply +8.48 ms; … + Equality/rebuild — net +1.18 ms +0.197% ◆ Rebuild +1.18 ms + ↳ Native rebuild replaced +1.18 ms ◆ Rebuild +1.18 ms + + Program + Equality account for +12.6% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + Ruleset drivers — herbie.egg + + Driver Δ Wall share Important phase changes + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Program rules — own work +147 ms +14.0% ◆ Search +94.5 ms; Apply +47.3 ms; … + ↳ ruleset-11 +22.7 ms ◆ Search +14.5 ms; Apply +7.27 ms; … + ↳ ruleset-10 +20.8 ms ◆ Search +13.3 ms; Apply +6.67 ms; … + ↳ ruleset-09 +18.9 ms ◆ Search +12.1 ms; Apply +6.06 ms; … + ↳ ruleset-08 +17.0 ms ◆ Search +10.9 ms; Apply +5.45 ms; … + ↳ ruleset-07 +15.1 ms ◆ Search +9.70 ms; Apply +4.85 ms; … + ↳ Other (7 more source rulesets) +52.9 ms ◆ Search +33.9 ms; Apply +17.0 ms; … + Equality/rebuild — net +1.97 ms +0.188% ◆ Rebuild +1.97 ms + ↳ Native rebuild replaced +1.97 ms ◆ Rebuild +1.97 ms + + Program + Equality account for +14.2% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. + Maintenance rules shown: none. + ──────────────────────────────────────────────── Slowdown decomposition ──────────────────────────────────────────────── + + File Wall Δ Typecheck Frontend Program Equality Commands Residual + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Suite total +1550 ms 0% 0 ms 0% 0 ms +10.7% +165 ms 0% 0 ms 0% 0 ms ◆ +89.3% +1385 ms + math.egg -200 ms 0% 0 ms 0% 0 ms +14.6% -29.1 ms +0.985% -1.97 ms 0% 0 ms ◆ +84.4% -169 ms + eggcc-extract.egg -150 ms 0% 0 ms 0% 0 ms +21.1% -31.7 ms +0.788% -1.18 ms 0% 0 ms ◆ +78.1% -117 ms + pointer-analysis- 0 ms — 0 ms — 0 ms — -15.3 ms — -0.394 ms — 0 ms — +15.7 ms + small.egg + hardboiled.egg +250 ms 0% 0 ms 0% 0 ms +8.00% +20.0 ms +0.158% +0.394 ms 0% 0 ms ◆ +91.8% +230 ms + luminal.egg +600 ms 0% 0 ms 0% 0 ms +12.4% +74.2 ms +0.197% +1.18 ms 0% 0 ms ◆ +87.4% +525 ms + herbie.egg +1050 ms 0% 0 ms 0% 0 ms +14.0% +147 ms +0.188% +1.97 ms 0% 0 ms ◆ +85.8% +901 ms + + Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes + parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets + except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes + actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold + type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and + interactive reports. Signed values carry the same information without styling. Residual is wall time minus every + recorded leaf; ! means an endpoint's mean residual is negative. ─────────────────────────────────────────────────── Per-file results ─────────────────────────────────────────────────── Wall time diff --git a/tests/report_fixtures.py b/tests/report_fixtures.py index ab2cfc9f..94135c79 100644 --- a/tests/report_fixtures.py +++ b/tests/report_fixtures.py @@ -3,13 +3,14 @@ from __future__ import annotations from pathlib import Path +from typing import Literal from benchmarking import models from benchmarking.reports.store import ( REPORT_SCHEMA_VERSION, ReportRecord, ReportStore, - RulesetTimingRecord, + TimingLeafRecord, TimingSummaryRecord, ) @@ -62,30 +63,54 @@ def make_record( def make_ruleset_timing( name: str = "rules", *, + assembly_ns: int = 0, search_ns: int = 400_000_000, apply_ns: int = 200_000_000, - unattributed_ns: int = 0, + execution_ns: int = 0, merge_ns: int = 200_000_000, rebuild_ns: int = 100_000_000, -) -> RulesetTimingRecord: + role: Literal["program", "equality"] = "program", +) -> tuple[TimingLeafRecord, ...]: """Construct one valid ruleset timing fixture.""" - return { - "name": name, - "search_ns": search_ns, - "apply_ns": apply_ns, - "unattributed_ns": unattributed_ns, - "merge_ns": merge_ns, - "rebuild_ns": rebuild_ns, - } - + responsibility = "equality" if role == "equality" else "program" + return ( + {"path": [responsibility, "assembly", name], "ns": assembly_ns}, + {"path": [responsibility, "search", name], "ns": search_ns}, + {"path": [responsibility, "apply", name], "ns": apply_ns}, + {"path": [responsibility, "execution", name], "ns": execution_ns}, + {"path": [responsibility, "merge", name], "ns": merge_ns}, + {"path": ["equality", "rebuild", name], "ns": rebuild_ns}, + ) -def make_timing_summary(*rulesets: RulesetTimingRecord) -> TimingSummaryRecord: - """Construct a valid v2 timing-summary fixture.""" +def make_timing_summary( + *rulesets: tuple[TimingLeafRecord, ...], + typecheck_ns: int = 0, + frontend_parse_ns: int = 0, + frontend_other_ns: int = 0, + frontend_install_ns: int = 0, + commands_actions_ns: int = 0, + commands_check_ns: int = 0, + commands_other_ns: int = 0, +) -> TimingSummaryRecord: + """Construct a valid V3 timing-summary fixture.""" + + timing_groups = rulesets or (make_ruleset_timing(),) + timings: list[TimingLeafRecord] = [ + {"path": ["typecheck", "total"], "ns": typecheck_ns}, + {"path": ["frontend", "parse"], "ns": frontend_parse_ns}, + {"path": ["frontend", "other"], "ns": frontend_other_ns}, + {"path": ["frontend", "install"], "ns": frontend_install_ns}, + {"path": ["commands", "actions"], "ns": commands_actions_ns}, + {"path": ["commands", "check"], "ns": commands_check_ns}, + {"path": ["commands", "other"], "ns": commands_other_ns}, + ] + timings.extend(leaf for group in timing_groups for leaf in group) + timings.sort(key=lambda leaf: leaf["path"]) return { - "schema_version": 2, - "rulesets": list(rulesets or (make_ruleset_timing(),)), + "schema_version": 3, + "timings": timings, } diff --git a/tests/test_collection.py b/tests/test_collection.py index e451802c..9de00cdf 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -455,16 +455,12 @@ def fake_run_command(command: list[str], checkout_path: Path, timeout_sec: int) summary_path.write_text( json.dumps( { - "schema_version": 2, - "rulesets": [ - { - "name": "rules", - "search_ns": 4, - "apply_ns": 6, - "unattributed_ns": 10, - "merge_ns": 20, - "rebuild_ns": 30, - } + "schema_version": 3, + "timings": [ + {"path": ["program", "search", "rules"], "ns": 4}, + {"path": ["program", "apply", "rules"], "ns": 6}, + {"path": ["program", "execution", "rules"], "ns": 10}, + {"path": ["program", "merge", "rules"], "ns": 20}, ], } ), @@ -480,10 +476,12 @@ def fake_run_command(command: list[str], checkout_path: Path, timeout_sec: int) assert "--proofs" not in commands[0] assert "--proofs" in commands[1] assert off.timing_summary is not None - assert off.timing_summary["rulesets"][0]["search_ns"] == 4 - assert off.timing_summary["rulesets"][0]["apply_ns"] == 6 - assert off.timing_summary["rulesets"][0]["unattributed_ns"] == 10 - assert off.timing_summary["rulesets"][0]["merge_ns"] == 20 + assert off.timing_summary["timings"] == [ + {"path": ["program", "search", "rules"], "ns": 4}, + {"path": ["program", "apply", "rules"], "ns": 6}, + {"path": ["program", "execution", "rules"], "ns": 10}, + {"path": ["program", "merge", "rules"], "ns": 20}, + ] assert proofs.timing_summary is not None diff --git a/tests/test_report_analysis.py b/tests/test_report_analysis.py index fca465fa..7e91e522 100644 --- a/tests/test_report_analysis.py +++ b/tests/test_report_analysis.py @@ -10,7 +10,7 @@ from benchmarking import models from benchmarking.reports.analysis import analyze_pair -from benchmarking.reports.store import ReportRecord, ReportStore +from benchmarking.reports.store import ReportRecord, ReportStore, TimingSummaryRecord from .report_fixtures import make_record, make_ruleset_timing, make_target, make_timing_summary, write_report @@ -68,10 +68,10 @@ def test_analysis_computes_only_the_requested_detail_rows(tmp_path: Path) -> Non rulesets = analyze_pair(store, comparison, "rulesets") assert len(summary.summary) == 5 - assert not summary.files and not summary.phases and not summary.rulesets - assert files.files and not files.phases and not files.rulesets - assert phases.files and phases.phases and not phases.rulesets - assert rulesets.files and rulesets.phases and rulesets.rulesets + assert not summary.files and not summary.decomposition and not summary.rulesets + assert files.files and not files.decomposition and not files.rulesets + assert phases.files and phases.decomposition and not phases.rulesets + assert rulesets.files and rulesets.decomposition and rulesets.rulesets def test_pair_statistics_and_fieller_intervals(tmp_path: Path) -> None: @@ -255,14 +255,14 @@ def test_valid_tail_does_not_inherit_an_unrelated_invalid_file_issue(tmp_path: P assert all(row.ratio.issue is None for row in tails) -def test_phase_rows_are_exhaustive_and_outside_is_wall_residual(tmp_path: Path) -> None: +def test_mechanism_buckets_are_additive_and_residual_closes_to_wall(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path) baseline_timing = make_timing_summary( make_ruleset_timing( search_ns=100, apply_ns=200, - unattributed_ns=17, + execution_ns=17, merge_ns=300, rebuild_ns=400, ) @@ -271,7 +271,7 @@ def test_phase_rows_are_exhaustive_and_outside_is_wall_residual(tmp_path: Path) make_ruleset_timing( search_ns=200, apply_ns=100, - unattributed_ns=23, + execution_ns=23, merge_ns=600, rebuild_ns=200, ) @@ -294,32 +294,114 @@ def test_phase_rows_are_exhaustive_and_outside_is_wall_residual(tmp_path: Path) ), ) - phases = analyze_pair(ReportStore(report), comparison, "phases").phases + suite, file_row = analyze_pair(ReportStore(report), comparison, "phases").decomposition - assert [row.phase for row in phases] == [ - "search", - "apply", - "unattributed", - "merge", - "rebuild", - "outside", - ] - assert [(row.baseline.timing.point, row.candidate.timing.point) for row in phases] == [ - (100.0, 200.0), - (200.0, 100.0), - (17.0, 23.0), - (300.0, 600.0), - (400.0, 200.0), - (483.0, 877.0), - ] - assert [row.delta_ns for row in phases] == [100.0, -100.0, 6.0, 300.0, -200.0, 394.0] - assert [row.wall_delta_contribution for row in phases] == pytest.approx([0.2, -0.2, 0.012, 0.6, -0.4, 0.788]) - assert sum(row.wall_delta_contribution or 0.0 for row in phases) == pytest.approx(1.0) - assert phases[0].baseline.wall_share == pytest.approx(100.0 / 1_500.0) - assert phases[-1].candidate.wall_share == pytest.approx(877.0 / 2_000.0) + assert suite.file_order is None + assert file_row.file_order == 0 + assert file_row.wall_delta_ns == pytest.approx(500.0) + assert [cell.delta_ns for cell in file_row.mechanisms] == pytest.approx([0.0, 0.0, 306.0, -200.0, 0.0, 394.0]) + assert [cell.slowdown_share for cell in file_row.mechanisms] == pytest.approx([0.0, 0.0, 0.612, -0.4, 0.0, 0.788]) + assert sum(cell.delta_ns or 0.0 for cell in file_row.mechanisms) == pytest.approx(file_row.wall_delta_ns) + assert sum(cell.slowdown_share or 0.0 for cell in file_row.mechanisms) == pytest.approx(1.0) + assert suite.wall_delta_ns == file_row.wall_delta_ns + assert suite.mechanisms == file_row.mechanisms + + +def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp_path: Path) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + timing = make_timing_summary( + make_ruleset_timing( + assembly_ns=31, + search_ns=37, + apply_ns=41, + execution_ns=43, + merge_ns=47, + rebuild_ns=53, + ), + frontend_parse_ns=11, + typecheck_ns=13, + frontend_other_ns=17, + frontend_install_ns=19, + commands_actions_ns=23, + commands_check_ns=7, + commands_other_ns=29, + ) + zero_timing = make_timing_summary( + make_ruleset_timing( + assembly_ns=0, + search_ns=0, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=0, + ) + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + wall_sec=0.000001, + timing_summary=zero_timing, + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + wall_sec=0.0000015, + timing_summary=timing, + ), + ) + + views = analyze_pair(ReportStore(report), comparison, "rulesets") + file_row = views.decomposition[1] + + assert file_row.wall_delta_ns == pytest.approx(500.0) + assert [cell.delta_ns for cell in file_row.mechanisms] == pytest.approx([13.0, 47.0, 199.0, 53.0, 59.0, 129.0]) + assert sum(cell.delta_ns or 0.0 for cell in file_row.mechanisms) == pytest.approx(500.0) + program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") + equality = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "equality") + native_rebuild = next(row for row in views.rulesets if row.kind == "native_rebuild") + assert program.delta.phases == pytest.approx((31, 37, 41, 43, 47, 0)) + assert program.delta.total == file_row.mechanisms.program.delta_ns == 199 + assert equality.delta.phases == pytest.approx((0, 0, 0, 0, 0, 53)) + assert equality.delta.total == file_row.mechanisms.equality.delta_ns == 53 + assert native_rebuild.delta == equality.delta + + +@pytest.mark.parametrize( + ("path", "message"), + ((["residual", "stored"], "residual is derived"), (["mystery", "work"], "unknown timing responsibility")), +) +def test_invalid_timing_responsibilities_are_rejected_by_the_reader( + tmp_path: Path, + path: list[str], + message: str, +) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + invalid = cast( + TimingSummaryRecord, + {"schema_version": 3, "timings": [{"path": path, "ns": 1}]}, + ) + write_report( + report, + make_record(0, started_at="2026-07-15T12:00:00Z", binary_sha256="sha256:baseline"), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + timing_summary=invalid, + ), + ) + with pytest.raises(ValueError, match=message): + analyze_pair(ReportStore(report), comparison, "phases") -def test_phase_endpoints_have_student_t_intervals_and_wall_context(tmp_path: Path) -> None: + +def test_mechanism_decomposition_uses_endpoint_means_and_wall_context(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path, rounds=2) records: list[ReportRecord] = [] @@ -341,27 +423,23 @@ def test_phase_endpoints_have_student_t_intervals_and_wall_context(tmp_path: Pat ) write_report(report, *records) - search = analyze_pair(ReportStore(report), comparison, "phases").phases[0] - half_width = 12.706204736432095 * 100.0 + file_row = analyze_pair(ReportStore(report), comparison, "phases").decomposition[1] - assert search.baseline.timing.point == 200 - assert search.baseline.timing.ci_low == pytest.approx(200 - half_width) - assert search.baseline.timing.ci_high == pytest.approx(200 + half_width) - assert search.baseline.wall_share == pytest.approx(200 / 1_100) - assert search.candidate.timing.point == 300 - assert search.candidate.wall_share == pytest.approx(300 / 1_600) - assert search.delta_ns == 100 - assert search.wall_delta_contribution == pytest.approx(0.2) + assert file_row.wall_delta_ns == pytest.approx(500.0) + assert file_row.mechanisms.program.delta_ns == 100 + assert file_row.mechanisms.program.slowdown_share == pytest.approx(0.2) + assert file_row.mechanisms.residual.delta_ns == 400 + assert file_row.mechanisms.residual.slowdown_share == pytest.approx(0.8) -def test_ruleset_union_distinguishes_absence_from_zero_and_aggregates_iterations(tmp_path: Path) -> None: +def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path, rounds=2) zero = make_ruleset_timing( "recorded-zero", search_ns=0, apply_ns=0, - unattributed_ns=0, + execution_ns=0, merge_ns=0, rebuild_ns=0, ) @@ -392,6 +470,14 @@ def test_ruleset_union_distinguishes_absence_from_zero_and_aggregates_iterations binary_sha256="sha256:candidate", timing_summary=make_timing_summary( make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing( + "assembly-only", + assembly_ns=5, + search_ns=0, + apply_ns=0, + merge_ns=0, + rebuild_ns=0, + ), zero, ), ), @@ -401,25 +487,149 @@ def test_ruleset_union_distinguishes_absence_from_zero_and_aggregates_iterations binary_sha256="sha256:candidate", timing_summary=make_timing_summary( make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing( + "assembly-only", + assembly_ns=5, + search_ns=0, + apply_ns=0, + merge_ns=0, + rebuild_ns=0, + ), zero, ), ), ) - rows = {row.name: row for row in analyze_pair(ReportStore(report), comparison, "rulesets").rulesets} + views = analyze_pair(ReportStore(report), comparison, "rulesets") + rows = {row.name: row for row in views.rulesets if row.kind == "ruleset"} - assert rows["baseline-only"].baseline == (10, 10, 10) - assert rows["baseline-only"].candidate is None - assert rows["candidate-only"].baseline is None - assert rows["candidate-only"].candidate == (20, 20, 20) - assert rows["sporadic"].baseline is not None - assert rows["sporadic"].baseline.point == 4 assert rows["baseline-only"].delta.phases.search == -10 assert rows["candidate-only"].delta.phases.search == 20 + assert rows["sporadic"].delta.phases.search == -4 + assert rows["assembly-only"].delta.phases.assembly == 5 + assert rows["assembly-only"].delta.total == 5 assert "recorded-zero" not in rows + program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") + assert program.ruleset_count == 4 + assert program.delta.total == 11 + + +def test_ruleset_drilldown_separates_program_work_from_native_rebuild(tmp_path: Path) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + baseline = cast( + TimingSummaryRecord, + { + "schema_version": 3, + "timings": [ + {"path": ["program", "search", "rules/λ"], "ns": 0}, + {"path": ["equality", "rebuild", "rules/λ"], "ns": 0}, + ], + }, + ) + candidate = cast( + TimingSummaryRecord, + { + "schema_version": 3, + "timings": [ + {"path": ["program", "search", "rules/λ"], "ns": 10}, + {"path": ["equality", "rebuild", "rules/λ"], "ns": 7}, + ], + }, + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + timing_summary=baseline, + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + timing_summary=candidate, + ), + ) + + rows = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets + program_rule = next(row for row in rows if row.kind == "ruleset") + native_rebuild = next(row for row in rows if row.kind == "native_rebuild") + + assert program_rule.name == "rules/λ" + assert program_rule.mechanism == "program" + assert program_rule.delta.phases.search == 10 + assert program_rule.delta.phases.rebuild == 0 + assert program_rule.delta.total == 10 + assert native_rebuild.mechanism == "equality" + assert native_rebuild.delta.phases.rebuild == 7 + assert native_rebuild.delta.total == 7 + + +def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: Path) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + candidate = make_timing_summary( + make_ruleset_timing( + "source", + assembly_ns=2, + search_ns=3, + apply_ns=5, + execution_ns=7, + merge_ns=11, + rebuild_ns=13, + ), + make_ruleset_timing( + "maintenance", + assembly_ns=17, + search_ns=19, + apply_ns=23, + execution_ns=29, + merge_ns=31, + rebuild_ns=37, + role="equality", + ), + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + timing_summary=cast(TimingSummaryRecord, {"schema_version": 3, "timings": []}), + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + timing_summary=candidate, + ), + ) + views = analyze_pair(ReportStore(report), comparison, "rulesets") + program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") + equality = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "equality") + maintenance = next( + row + for row in views.rulesets + if row.kind == "ruleset" and row.mechanism == "equality" and row.name == "maintenance" + ) + native_rebuild = next(row for row in views.rulesets if row.kind == "native_rebuild") + mechanisms = views.decomposition[1].mechanisms -def test_ruleset_presentation_is_fixed_top_ten_by_absolute_delta_then_name(tmp_path: Path) -> None: + assert program.delta.total == mechanisms.program.delta_ns == 28 + assert maintenance.delta.total == 156 + assert native_rebuild.delta.total == 13 + assert equality.delta.total == mechanisms.equality.delta_ns == 169 + assert maintenance.delta.total + native_rebuild.delta.total == equality.delta.total + for parent in (program, equality): + children = [row for row in views.rulesets if row.kind != "aggregate" and row.mechanism == parent.mechanism] + assert sum(row.delta.total for row in children) == parent.delta.total + assert all(sum(row.delta.phases[index] for row in children) == parent.delta.phases[index] for index in range(6)) + + +def test_program_children_are_fixed_top_five_plus_exact_per_group_other(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path) names = tuple(reversed(tuple(f"rules-{index:02d}" for index in range(12)))) @@ -447,9 +657,91 @@ def test_ruleset_presentation_is_fixed_top_ten_by_absolute_delta_then_name(tmp_p rulesets = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets - assert len(rulesets) == 10 - assert [row.name for row in rulesets] == [f"rules-{index:02d}" for index in range(10)] - assert {row.ruleset_count for row in rulesets} == {12} + parents = [row for row in rulesets if row.kind == "aggregate"] + contributors = [row for row in rulesets if row.kind == "ruleset"] + other = next(row for row in rulesets if row.kind == "other") + assert [(row.mechanism, row.ruleset_count, row.delta.total) for row in parents] == [ + ("program", 12, 12), + ("equality", 0, 0), + ] + assert [row.name for row in contributors] == [f"rules-{index:02d}" for index in range(5)] + assert all(row.mechanism == "program" for row in contributors) + assert other.ruleset_count == 7 + assert other.delta.total == 7 + assert other.delta.phases.search == 7 + program_parent = parents[0] + assert sum(row.delta.total for row in contributors) + other.delta.total == program_parent.delta.total + assert all( + sum(row.delta.phases[index] for row in contributors) + other.delta.phases[index] + == program_parent.delta.phases[index] + for index in range(6) + ) + + +def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tmp_path: Path) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + names = tuple(f"maintenance-{index}" for index in range(7)) + source = make_ruleset_timing( + "source", + assembly_ns=0, + search_ns=0, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=13, + ) + baseline_maintenance = tuple( + make_ruleset_timing( + name, + assembly_ns=0, + search_ns=0, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=0, + role="equality", + ) + for name in names + ) + candidate_maintenance = tuple( + make_ruleset_timing( + name, + assembly_ns=0, + search_ns=index + 1, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=0, + role="equality", + ) + for index, name in enumerate(names) + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + timing_summary=make_timing_summary(source, *baseline_maintenance), + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + timing_summary=make_timing_summary(source, *candidate_maintenance), + ), + ) + + rulesets = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets + maintenance = [row for row in rulesets if row.kind == "ruleset" and row.mechanism == "equality"] + equality = next(row for row in rulesets if row.kind == "aggregate" and row.mechanism == "equality") + + assert len(maintenance) == 7 + assert [row.name for row in maintenance] == list(reversed(names)) + assert equality.delta.total == sum(range(1, 8)) + assert not any(row.kind == "native_rebuild" for row in rulesets) + assert not any(row.kind == "other" and row.mechanism == "equality" for row in rulesets) def _fieller_bounds( diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index bbf3b7f5..a36d75d3 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -6,19 +6,22 @@ from typing import cast from pytest import MonkeyPatch +from rich import box from rich.cells import cell_len from rich.console import Console from rich.rule import Rule from syrupy.assertion import SnapshotAssertion from benchmarking import models +from benchmarking.reports.analysis import PhaseValues, RulesetDelta from benchmarking.reports.catalog import ReportCatalog, ReportMessage, ReportTable, report_id from benchmarking.reports.presentation import ( + _important_phase_changes, build_report_catalog, format_duration, report_file_labels, ) -from benchmarking.reports.render import render_markdown_report_document, render_rich_report_document +from benchmarking.reports.render import render_markdown_report_document, render_rich_report_document, render_rich_table from benchmarking.reports.store import ReportRecord, ReportStore from .report_fixtures import make_endpoint, make_record, make_ruleset_timing, make_timing_summary, write_report @@ -86,6 +89,7 @@ def test_shared_formatters_keep_compact_units_and_unambiguous_paths() -> None: def test_rich_report_is_readable_at_realistic_widths(tmp_path: Path) -> None: report_path, comparison = _six_file_pair_case(tmp_path) catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") + ellipsis_count: int | None = None for width in (80, 119, 120, 160, 200): console = Console(record=True, width=width, color_system=None) @@ -95,16 +99,25 @@ def test_rich_report_is_readable_at_realistic_widths(tmp_path: Path) -> None: assert rendered.count("Warning: detailed Rich report") == (1 if width < 120 else 0) assert max(cell_len(line) for line in rendered.splitlines()) <= width rule_lines = tuple(line for line in rendered.splitlines() if "─" in line) - assert len(rule_lines) == 5 assert all( any(title in line for line in rule_lines) - for title in ("Ruleset comparison", "Phase comparison", "Per-file results", "Comparison", "Summary —") + for title in ( + "Ruleset drivers", + "Slowdown decomposition", + "Per-file results", + "Comparison", + "Summary —", + ) ) - assert rendered.index("Ruleset comparison") < rendered.index("Phase comparison") - assert rendered.index("Phase comparison") < rendered.index("Per-file results") + assert rendered.index("Ruleset drivers") < rendered.index("Slowdown decomposition") + assert rendered.index("Slowdown decomposition") < rendered.index("Per-file results") assert rendered.index("Per-file results") < rendered.index("Comparison") assert rendered.index("Comparison") < rendered.rindex("Summary —") - assert "…" not in rendered + if ellipsis_count is None: + ellipsis_count = rendered.count("…") + assert ellipsis_count > 0 + else: + assert rendered.count("…") == ellipsis_count assert "Per-file wall time" not in rendered assert "Benchmark summary" not in rendered assert "math.egg" in rendered @@ -131,10 +144,10 @@ def test_realistic_six_file_rich_120_snapshot( rendered = console.export_text() assert rendered == snapshot - assert rendered.count("Ruleset comparison —") == 6 - assert rendered.count("Phase comparison —") == 6 + assert rendered.count("Ruleset drivers —") == 6 + assert rendered.count("Slowdown decomposition") >= 1 assert "Warning: detailed Rich report" not in rendered - assert "…" not in rendered + assert "Other (7 more source rulesets)" in rendered def test_detail_level_is_cumulative(tmp_path: Path) -> None: @@ -155,20 +168,113 @@ def test_detail_level_is_cumulative(tmp_path: Path) -> None: assert tuple(section.id for section in catalog.sections) == section_ids -def test_phase_detail_has_one_six_row_table_per_file_and_one_guide(tmp_path: Path) -> None: +def test_all_rich_tables_use_one_compact_style(tmp_path: Path) -> None: + report_path, comparison = _pair_case(tmp_path) + catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") + tables = [ + render_rich_table(block) + for section in catalog.sections + for block in section.blocks + if isinstance(block, ReportTable) + ] + + assert tables + assert all(table.box is box.SIMPLE_HEAVY and not table.show_lines for table in tables) + + +def test_phase_detail_is_one_additive_decomposition_table(tmp_path: Path) -> None: report_path, comparison = _pair_case(tmp_path) catalog = build_report_catalog(ReportStore(report_path), comparison, "phases") section = next(section for section in catalog.sections if section.id == "phases") - assert isinstance(section.blocks[0], ReportMessage) tables = tuple(block for block in section.blocks if isinstance(block, ReportTable)) - assert len(tables) == len(comparison.files) - expected_columns = ("phase", "baseline", "candidate", "delta", "wall_delta") - assert all(tuple(column.id for column in table.columns) == expected_columns for table in tables) - assert all(len(table.rows) == 6 for table in tables) + assert len(tables) == 1 + table = tables[0] + assert tuple(column.id for column in table.columns) == ( + "file", + "wall_delta", + "typecheck", + "frontend", + "program", + "equality", + "commands", + "residual", + ) + assert len(table.rows) == len(comparison.files) + 1 + assert table.rows[0].cells[0].display == "Suite total" + assert [row.cells[0].display for row in table.rows[1:]] == ["math.egg", "rewrite.egg"] + assert table.columns[3].label == "Frontend" + assert table.columns[4].label == "Program" + assert table.columns[5].label == "Equality" + assert table.caption is not None and "candidate − baseline" in table.caption + assert all("%" in cell.display.partition(" ")[0] for cell in table.rows[0].cells[2:]) + assert all(sum("◆" in cell.display for cell in row.cells[2:]) == 1 for row in table.rows) + assert table.rows[0].cells[2].tone == "muted" + assert table.rows[0].cells[4].tone == "emphasis" + assert table.rows[1].cells[1].tone == "positive" + assert table.rows[1].cells[4].tone == "emphasis" + assert table.rows[1].cells[7].tone == "positive" + + +def test_ruleset_detail_unfolds_program_and_equality_with_explicit_children(tmp_path: Path) -> None: + report_path, comparison = _pair_case(tmp_path) + catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") + + section = next(section for section in catalog.sections if section.id == "rulesets") + guide = section.blocks[0] + assert isinstance(guide, ReportMessage) + assert "Parent rows exactly match" in guide.text + assert "one global Native rebuild replaced row" in guide.text + assert "top 5 plus an exact per-group Other" in guide.text + assert "↳ marks children" in guide.text + assert "max(1 ms, 10% of |row Δ|)" in guide.text + + math = next( + block for block in section.blocks if isinstance(block, ReportTable) and block.title.endswith("math.egg") + ) + rewrite = next( + block for block in section.blocks if isinstance(block, ReportTable) and block.title.endswith("rewrite.egg") + ) + assert tuple(column.id for column in math.columns) == ("driver", "delta", "share", "important_phases") + assert math.columns[2].label == "Wall share" + assert [row.cells[0].display for row in math.rows] == [ + "Program rules — own work", + "↳ simplify", + "↳ finish", + "Equality/rebuild — net", + ] + assert math.rows[0].cells[0].tone == "emphasis" + assert math.rows[0].cells[1].tone == "positive" + assert math.rows[0].cells[3].display == "◆ Search -80.0 ms; Apply -36.0 ms" + assert math.rows[0].cells[2].display == "+58.0%" + assert math.rows[1].cells[2].display == "" + assert math.rows[3].cells[0].tone == "emphasis" + assert math.rows[3].cells[3].display == "0 ns" + assert math.caption is not None and "Program + Equality account for +58.0%" in math.caption + assert rewrite.rows[0].cells[1].tone == "default" + + +def test_ratio_tones_use_green_for_improvements_and_dim_unclear_results(tmp_path: Path) -> None: + report_path, comparison = _pair_case(tmp_path) + catalog = build_report_catalog(ReportStore(report_path), comparison) + summary = next(section for section in catalog.sections if section.id == "summary") + table = next(block for block in summary.blocks if isinstance(block, ReportTable)) + expected = { + "higher": "default", + "invalid": "error", + "lower": "positive", + "point_only": "muted", + "unclear": "muted", + } + + for row in table.rows: + result = row.cells[4].raw + assert isinstance(result, str) + assert row.cells[3].tone == expected[result] + assert row.cells[4].tone == expected[result] -def test_negative_outside_residual_keeps_an_explicit_warning(tmp_path: Path) -> None: +def test_negative_residual_keeps_an_explicit_warning(tmp_path: Path) -> None: report_path = tmp_path / "negative-residual.jsonl" file = models.FileSpec("file.egg", tmp_path / "file.egg", "sha256:file") baseline = make_endpoint(binary_sha256="sha256:baseline", treatment="off") @@ -210,44 +316,16 @@ def test_negative_outside_residual_keeps_an_explicit_warning(tmp_path: Path) -> markdown = render_markdown_report_document(build_report_catalog(ReportStore(report_path), comparison, "phases")) - assert "!-200 ms · -20.0%" in markdown - assert "! marks a negative residual" in markdown + assert "!◆ +150% +300 ms" in markdown + assert "! means an endpoint's mean residual is negative" in markdown -def test_ruleset_display_distinguishes_absent_from_measured_zero(tmp_path: Path) -> None: - report_path = tmp_path / "ruleset-presence.jsonl" - file = models.FileSpec("file.egg", tmp_path / "file.egg", "sha256:file") - baseline = make_endpoint(binary_sha256="sha256:baseline", treatment="off") - candidate = make_endpoint(binary_sha256="sha256:candidate", treatment="proofs") - write_report( - report_path, - make_record( - 0, - started_at="2026-07-17T12:00:00Z", - binary_sha256=baseline.target.binary_sha256, - treatment=baseline.treatment, - timing_summary=make_timing_summary( - make_ruleset_timing("measured-zero", search_ns=0, apply_ns=0, merge_ns=0, rebuild_ns=0), - make_ruleset_timing("removed", search_ns=10, apply_ns=0, merge_ns=0, rebuild_ns=0), - ), - ), - make_record( - 1, - started_at="2026-07-17T12:00:01Z", - binary_sha256=candidate.target.binary_sha256, - treatment=candidate.treatment, - timing_summary=make_timing_summary( - make_ruleset_timing("measured-zero", search_ns=5, apply_ns=0, merge_ns=0, rebuild_ns=0), - make_ruleset_timing("added", search_ns=20, apply_ns=0, merge_ns=0, rebuild_ns=0), - ), - ), - ) - comparison = models.ComparisonSpec(baseline, candidate, (file,), 1, 120) +def test_important_phase_changes_use_the_documented_deterministic_threshold() -> None: + delta = RulesetDelta(20_000_000, PhaseValues(500_000, 10_000_000, 3_000_000, 2_000_000, 4_000_000, 500_000)) - markdown = render_markdown_report_document(build_report_catalog(ReportStore(report_path), comparison, "rulesets")) - - assert "| measured-zero | 0 ns | 5.00 ns | +5.00 ns |" in markdown - assert "| added | — | 20.0 ns | +20.0 ns |" in markdown + assert _important_phase_changes(delta) == ( + "◆ Search +10.0 ms; Apply +3.00 ms; Execution +2.00 ms; Merge +4.00 ms; …" + ) def test_one_file_summary_removes_redundant_wall_and_rss_tails(tmp_path: Path) -> None: @@ -336,11 +414,16 @@ def test_timed_out_file_has_missing_phase_cells_and_ruleset_status(tmp_path: Pat catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") phase_section = next(section for section in catalog.sections if section.id == "phases") phase_table = next(block for block in phase_section.blocks if isinstance(block, ReportTable)) - candidate_column = next(index for index, column in enumerate(phase_table.columns) if column.id == "candidate") - assert all(row.cells[candidate_column].display == "—" for row in phase_table.rows) + assert len(phase_table.rows) == 2 + assert all(cell.display == "—" for row in phase_table.rows for cell in row.cells[1:]) ruleset_section = next(section for section in catalog.sections if section.id == "rulesets") assert isinstance(ruleset_section.blocks[0], ReportMessage) assert ruleset_section.blocks[0].text == "Status: timeout row selected" + summary_section = next(section for section in catalog.sections if section.id == "summary") + summary_table = next(block for block in summary_section.blocks if isinstance(block, ReportTable)) + invalid = next(row for row in summary_table.rows if row.cells[4].raw == "invalid") + assert invalid.cells[3].tone == "error" + assert invalid.cells[4].tone == "error" def _pair_case(tmp_path: Path) -> tuple[Path, models.ComparisonSpec]: @@ -428,7 +511,7 @@ def _six_file_pair_case(tmp_path: Path) -> tuple[Path, models.ComparisonSpec]: f"ruleset-{ruleset_order:02d}", search_ns=int((ruleset_order + 1) * (file_order + 1) * 2_000_000 * timing_factor), apply_ns=int((ruleset_order + 1) * (file_order + 1) * 1_000_000 * timing_factor), - unattributed_ns=int((ruleset_order + 1) * 200_000 * timing_factor), + execution_ns=int((ruleset_order + 1) * 200_000 * timing_factor), merge_ns=int((ruleset_order + 1) * 500_000 * timing_factor), rebuild_ns=int((ruleset_order + 1) * 250_000 * timing_factor), ) diff --git a/tests/test_report_store.py b/tests/test_report_store.py index ef2f41ca..9284a885 100644 --- a/tests/test_report_store.py +++ b/tests/test_report_store.py @@ -12,7 +12,7 @@ CacheKey, ReportRecord, ReportStore, - RulesetTimingRecord, + TimingLeafRecord, TimingSummaryRecord, parse_report_record, ) @@ -85,7 +85,7 @@ def test_typed_dict_schema_and_nested_values_round_trip(tmp_path: Path) -> None: "rules/λ", search_ns=6, apply_ns=5, - unattributed_ns=4, + execution_ns=4, merge_ns=7, rebuild_ns=3, ) @@ -99,11 +99,12 @@ def test_typed_dict_schema_and_nested_values_round_trip(tmp_path: Path) -> None: assert tuple(loaded) == tuple(ReportRecord.__annotations__) summary = cast(TimingSummaryRecord, loaded["timing_summary"]) assert tuple(summary) == tuple(TimingSummaryRecord.__annotations__) - rulesets = cast(list[RulesetTimingRecord], summary["rulesets"]) - assert tuple(rulesets[0]) == tuple(RulesetTimingRecord.__annotations__) + timings = cast(list[TimingLeafRecord], summary["timings"]) + assert all(tuple(leaf) == tuple(TimingLeafRecord.__annotations__) for leaf in timings) + assert ["program", "search", "rules/λ"] in [leaf["path"] for leaf in timings] -@pytest.mark.parametrize("schema_version", [None, 1], ids=["missing", "wrong"]) +@pytest.mark.parametrize("schema_version", [None, 2], ids=["missing", "wrong"]) def test_incompatible_report_schema_fails_during_load(tmp_path: Path, schema_version: int | None) -> None: report = tmp_path / "report.jsonl" current = make_record(0, started_at="2026-07-15T12:00:00Z") @@ -124,7 +125,7 @@ def test_incompatible_report_shapes_fail_during_load(tmp_path: Path, mixed: bool current = make_record(0, started_at="2026-07-15T12:00:00Z") old = cast(dict[str, object], make_record(1, started_at="2026-07-15T12:00:01Z")) timing = cast(dict[str, object], old["timing_summary"]) - timing["schema_version"] = 1 + timing["schema_version"] = 2 records = (current, old) if mixed else (old,) report.write_text("".join(f"{json.dumps(record)}\n" for record in records), encoding="utf-8") From d60202f644249ef565de0ca7c51871fe497440a2 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Wed, 12 Aug 2026 23:31:38 -0400 Subject: [PATCH 2/9] Add optimization ceilings to timing reports --- README.md | 13 ++- benchmarking/reports/analysis.py | 108 +++++++++++++++++- benchmarking/reports/presentation.py | 51 ++++++++- egg-math-benchmark/src/main.rs | 60 ++++++---- term-encoding-overhead-breakdown.md | 11 +- .../__snapshots__/test_report_rendering.ambr | 33 ++++++ tests/test_report_analysis.py | 78 +++++++++++++ tests/test_report_rendering.py | 26 ++++- 8 files changed, 342 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 4ea6056e..c7665a56 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ path. | --- | --- | | `summary` | comparison selection and headline summary | | `files` | per-file wall time and peak RSS estimates | -| `phases` | one additive slowdown decomposition across files and mechanisms | +| `phases` | additive slowdown decomposition plus suite optimization ceilings | | `rulesets` | Program/Equality driver groups and changed rulesets per file | The default is `summary`. For example: @@ -380,7 +380,7 @@ Residual is derived per observation as external wall time minus every recorded leaf. It includes process setup, reporting, teardown, and any still- uninstrumented work. -At `--detail phases`, one additive slowdown-decomposition table has a suite row +At `--detail phases`, the additive slowdown-decomposition table has a suite row and one row per file. Its rendered headers are `Wall Δ`, `Typecheck`, `Frontend`, `Program`, `Equality`, `Commands`, and `Residual`. Every mechanism cell displays its share of the wall-time change first, then @@ -392,6 +392,15 @@ measurements. Percentages may be negative or exceed 100% when mechanisms offset. `!` on a Residual cell means at least one endpoint's mean recorded total exceeded its wall time. +A compact suite-level `Optimization ceilings` table then resets selected +candidate-minus-baseline deltas to zero and reports the remaining wall-time +change and implied point ratio. It distinguishes removing only Equality +ruleset assembly from making the entire net Equality/rebuild responsibility +match the baseline. These rows are optimistic additive accounting bounds, not +predictions: they hold every other measured mean fixed, omit confidence +intervals, and cannot model interactions between optimizations. Residual is +never treated as removable work. + At `--detail rulesets`, one compact driver table appears per file. Its `Program rules — own work` and `Equality/rebuild — net` parent rows exactly match the corresponding cells in the decomposition, show their wall share, diff --git a/benchmarking/reports/analysis.py b/benchmarking/reports/analysis.py index c8553147..ac428934 100644 --- a/benchmarking/reports/analysis.py +++ b/benchmarking/reports/analysis.py @@ -24,6 +24,16 @@ RulesetPhaseName = Literal["assembly", "search", "apply", "execution", "merge", "rebuild"] RulesetMechanism = Literal["program", "equality"] RulesetRowKind = Literal["aggregate", "ruleset", "native_rebuild", "other"] +OptimizationScenario = Literal[ + "typecheck", + "frontend", + "frontend_and_typecheck", + "equality_assembly", + "equality", + "program", + "non_program", + "all_recorded", +] type _MetricKey = tuple[int, int, MetricName] type _ObservationKey = tuple[int, int] @@ -142,12 +152,22 @@ class RulesetContributorView(NamedTuple): delta: RulesetDelta +class OptimizationCeilingView(NamedTuple): + """One suite-wide accounting counterfactual with no causal-speedup claim.""" + + scenario: OptimizationScenario + reset_delta_ns: float + remaining_delta_ns: float + counterfactual_ratio: float + + class PairReportViewData(NamedTuple): """Typed analysis collections requested by one cumulative detail level.""" summary: tuple[SummaryView, ...] files: tuple[FileComparisonView, ...] decomposition: tuple[SlowdownDecompositionView, ...] + ceilings: tuple[OptimizationCeilingView, ...] rulesets: tuple[RulesetContributorView, ...] @@ -181,16 +201,17 @@ def analyze_pair( summary = _summary_rows(comparison, estimates, file_rows, t_critical) if detail == "summary": - return PairReportViewData(summary, (), (), ()) + return PairReportViewData(summary, (), (), (), ()) if detail == "files": - return PairReportViewData(summary, file_rows, (), ()) + return PairReportViewData(summary, file_rows, (), (), ()) timing = _timing_aggregates(observations) decomposition = _slowdown_decomposition(comparison, timing, issues, estimates) + ceilings = _optimization_ceilings(comparison, timing, estimates, decomposition) if detail == "phases": - return PairReportViewData(summary, file_rows, decomposition, ()) + return PairReportViewData(summary, file_rows, decomposition, ceilings, ()) rulesets = _ruleset_contributors(comparison, timing, issues) - return PairReportViewData(summary, file_rows, decomposition, rulesets) + return PairReportViewData(summary, file_rows, decomposition, ceilings, rulesets) def _selected_observations( @@ -435,6 +456,85 @@ def _slowdown_decomposition( return (suite, *result) +def _optimization_ceilings( + comparison: ComparisonSpec, + timing: dict[_ObservationKey, _TimingAggregate], + metric_estimates: dict[_MetricKey, _MetricEstimate], + decomposition: tuple[SlowdownDecompositionView, ...], +) -> tuple[OptimizationCeilingView, ...]: + """Reset selected suite deltas to zero as optimistic accounting bounds.""" + + scenarios: tuple[OptimizationScenario, ...] = ( + "typecheck", + "frontend", + "frontend_and_typecheck", + "equality_assembly", + "equality", + "program", + "non_program", + "all_recorded", + ) + if decomposition[0].issue is not None: + return () + baseline_points = [ + metric_estimates[(0, file_order, "wall_sec")].estimate.point for file_order in range(len(comparison.files)) + ] + candidate_points = [ + metric_estimates[(1, file_order, "wall_sec")].estimate.point for file_order in range(len(comparison.files)) + ] + if None in baseline_points or None in candidate_points: + return () + + baseline_wall_ns = math.fsum(cast(float, point) for point in baseline_points) * 1_000_000_000.0 + candidate_wall_ns = math.fsum(cast(float, point) for point in candidate_points) * 1_000_000_000.0 + if baseline_wall_ns <= 0 or candidate_wall_ns <= baseline_wall_ns: + return () + + equality_assembly_delta = 0.0 + for file_order in range(len(comparison.files)): + endpoint_means = [] + for endpoint_order in (0, 1): + aggregate = timing[(endpoint_order, file_order)] + paths = [ + path for path in aggregate.paths if len(path) >= 2 and path[0] == "equality" and path[1] == "assembly" + ] + endpoint_means.append(statistics.fmean(_sum_path_samples(aggregate, paths))) + equality_assembly_delta += endpoint_means[1] - endpoint_means[0] + + suite = decomposition[0] + deltas = suite.mechanisms + mechanism_deltas = (deltas.typecheck, deltas.frontend, deltas.program, deltas.equality, deltas.commands) + if any(cell.delta_ns is None for cell in mechanism_deltas): + return () + typecheck = cast(float, deltas.typecheck.delta_ns) + frontend = cast(float, deltas.frontend.delta_ns) + program = cast(float, deltas.program.delta_ns) + equality = cast(float, deltas.equality.delta_ns) + commands = cast(float, deltas.commands.delta_ns) + positive = tuple(max(delta, 0.0) for delta in (typecheck, frontend, program, equality, commands)) + typecheck_added, frontend_added, program_added, equality_added, commands_added = positive + reset_deltas = ( + typecheck_added, + frontend_added, + typecheck_added + frontend_added, + max(equality_assembly_delta, 0.0), + equality_added, + program_added, + typecheck_added + frontend_added + equality_added + commands_added, + math.fsum(positive), + ) + wall_delta = candidate_wall_ns - baseline_wall_ns + return tuple( + OptimizationCeilingView( + scenario, + reset_delta, + wall_delta - reset_delta, + (candidate_wall_ns - reset_delta) / baseline_wall_ns, + ) + for scenario, reset_delta in zip(scenarios, reset_deltas, strict=True) + ) + + def _timing_aggregates( observations: dict[_ObservationKey, tuple[IndexedRecord, ...]], ) -> dict[_ObservationKey, _TimingAggregate]: diff --git a/benchmarking/reports/presentation.py b/benchmarking/reports/presentation.py index 3078cf5e..cae73e58 100644 --- a/benchmarking/reports/presentation.py +++ b/benchmarking/reports/presentation.py @@ -19,6 +19,7 @@ Estimate, FileComparisonView, MetricName, + OptimizationCeilingView, PairReportViewData, RatioEstimate, ResultClass, @@ -73,6 +74,13 @@ "Important phases include every |phase Δ| ≥ max(1 ms, 10% of |row Δ|), always include the dominant phase (◆), " "and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases." ) +OPTIMIZATION_CAPTION = ( + "Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other " + "measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while " + "net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting " + "bounds, not implementation predictions; point ratios have no confidence intervals and interactions can " + "invalidate them. Residual is never removed." +) def build_report_catalog( @@ -92,7 +100,7 @@ def build_report_catalog( if _includes(detail, "files"): sections.append(_files_section(views.files, comparison, file_labels)) if _includes(detail, "phases"): - sections.append(_phases_section(views.decomposition, comparison, file_labels)) + sections.append(_phases_section(views.decomposition, views.ceilings, comparison, file_labels)) if _includes(detail, "rulesets"): sections.append(_rulesets_section(views, comparison, file_labels)) return ReportCatalog(tuple(sections)) @@ -319,6 +327,7 @@ def _files_section( def _phases_section( rows: Sequence[SlowdownDecompositionView], + ceilings: Sequence[OptimizationCeilingView], comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: @@ -373,7 +382,45 @@ def _phases_section( caption=DECOMPOSITION_CAPTION, alignments=("left", "right", "right", "right", "right", "right", "right", "right"), ) - return ReportSection("phases", "Slowdown decomposition", (table,)) + scenario_labels = { + "typecheck": "Remove added typechecking time", + "frontend": "Remove added frontend/install time", + "frontend_and_typecheck": "Remove added typechecking + frontend time", + "equality_assembly": "Remove added Equality assembly time", + "equality": "Remove added net Equality/rebuild time", + "program": "Remove added source-rule execution time", + "non_program": "Remove every added non-program mechanism", + "all_recorded": "Remove every recorded added mechanism", + } + ceiling_rows = tuple( + _row( + report_id("row", "phases", "ceiling", row.scenario), + text_cell(row.scenario, scenario_labels[row.scenario]), + text_cell(row.reset_delta_ns, format_duration(row.reset_delta_ns, signed=True)), + text_cell( + row.remaining_delta_ns, + _format_delta_ms(row.remaining_delta_ns), + tone=_delta_tone(row.remaining_delta_ns), + ), + text_cell( + row.counterfactual_ratio, + f"{_three_significant_digits(row.counterfactual_ratio)}x", + tone="positive" if row.counterfactual_ratio < 1 else "default", + ), + ) + for row in ceilings + ) + ceiling_table = _table( + report_id("table", "phases", "optimization-ceilings"), + "Optimization ceilings", + ("scenario", "reset_delta", "remaining_delta", "counterfactual_ratio"), + ("Hypothetical change", "Time removed", "Remaining wall Δ", "Implied ratio"), + ceiling_rows, + caption=OPTIMIZATION_CAPTION, + alignments=("left", "right", "right", "right"), + ) + blocks = (table, ceiling_table) if ceilings else (table,) + return ReportSection("phases", "Slowdown decomposition", blocks) def _slowdown_cell(cell: SlowdownCell, *, leader: bool, warning: bool) -> ReportCell: diff --git a/egg-math-benchmark/src/main.rs b/egg-math-benchmark/src/main.rs index 31b5e69f..188b6299 100644 --- a/egg-math-benchmark/src/main.rs +++ b/egg-math-benchmark/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result, ensure}; use clap::{Parser, ValueEnum}; use egg::{RecExpr, Runner, SimpleScheduler, StopReason}; -use egglog_reports::{RulesetTimingV2, TimingSummaryV2}; +use egglog_reports::TimingSummaryV3; use std::{ fs::File, io::BufWriter, @@ -44,7 +44,7 @@ fn main() -> Result<()> { Ok(()) } -fn run_math(proof_mode: ProofMode) -> Result<(TimingSummaryV2, usize)> { +fn run_math(proof_mode: ProofMode) -> Result<(TimingSummaryV3, usize)> { let left: RecExpr = CHECK_LEFT.parse().expect("fixed left check must parse"); let right: RecExpr = CHECK_RIGHT.parse().expect("fixed right check must parse"); let rules = math::rules(); @@ -89,36 +89,54 @@ fn run_math(proof_mode: ProofMode) -> Result<(TimingSummaryV2, usize)> { ); let proof_postprocessing_started = Instant::now(); - let proof_postprocessing_ns = if matches!(proof_mode, ProofMode::Extract | ProofMode::Check) { + let proof_postprocessing = if matches!(proof_mode, ProofMode::Extract | ProofMode::Check) { let mut explanation = runner.explain_equivalence(&left, &right); explanation.make_flat_explanation(); if proof_mode == ProofMode::Check { explanation.check_proof(&rules); } - duration_to_ns(proof_postprocessing_started.elapsed()) + proof_postprocessing_started.elapsed() } else { - 0 + Duration::ZERO }; - let timing = TimingSummaryV2 { - schema_version: 2, - rulesets: vec![RulesetTimingV2 { - name: String::new(), - search_ns: seconds_to_ns(report.search_time), - apply_ns: seconds_to_ns(report.apply_time), - unattributed_ns: seconds_to_ns( + let timing = TimingSummaryV3::new([ + ( + vec!["program".into(), "assembly".into(), String::new()], + Duration::ZERO, + ), + ( + vec!["program".into(), "search".into(), String::new()], + Duration::from_nanos(seconds_to_ns(report.search_time)), + ), + ( + vec!["program".into(), "apply".into(), String::new()], + Duration::from_nanos(seconds_to_ns(report.apply_time)), + ), + ( + vec!["program".into(), "execution".into(), String::new()], + Duration::from_nanos(seconds_to_ns( (report.total_time - report.search_time - report.apply_time - report.rebuild_time) .max(0.0), - ) - .saturating_add(proof_postprocessing_ns), - merge_ns: 0, - rebuild_ns: seconds_to_ns(report.rebuild_time), - }], - }; + )), + ), + ( + vec!["program".into(), "merge".into(), String::new()], + Duration::ZERO, + ), + ( + vec!["equality".into(), "rebuild".into(), String::new()], + Duration::from_nanos(seconds_to_ns(report.rebuild_time)), + ), + ( + vec!["commands".into(), "other".into()], + proof_postprocessing, + ), + ]); Ok((timing, report.egraph_nodes)) } -fn write_timing_summary(path: &Path, timing: &TimingSummaryV2) -> Result<()> { +fn write_timing_summary(path: &Path, timing: &TimingSummaryV3) -> Result<()> { let file = File::create(path) .with_context(|| format!("failed to create timing summary {}", path.display()))?; serde_json::to_writer(BufWriter::new(file), timing) @@ -132,10 +150,6 @@ fn seconds_to_ns(seconds: f64) -> u64 { (seconds * 1_000_000_000.0).min(u64::MAX as f64) as u64 } -fn duration_to_ns(duration: Duration) -> u64 { - duration.as_nanos().min(u128::from(u64::MAX)) as u64 -} - #[cfg(test)] mod tests { use super::*; diff --git a/term-encoding-overhead-breakdown.md b/term-encoding-overhead-breakdown.md index 3bb610d3..2df861f2 100644 --- a/term-encoding-overhead-breakdown.md +++ b/term-encoding-overhead-breakdown.md @@ -428,9 +428,12 @@ while its independent default-ruleset Search means are 60.349 ms and 84.870 ms. Herbie records 0.175 ms and 0.241 ms for checks. Check evaluation is therefore visible without being mistaken for transformed program-rule Search. -At `--detail phases`, presentation starts with one additive -slowdown-decomposition table with a Suite row and one row per file. At -`--detail rulesets`, one driver panel per file unfolds exactly the Program and +At `--detail phases`, presentation starts with an additive +slowdown-decomposition table with a Suite row and one row per file, followed +by a compact suite-level table of optimistic accounting ceilings. The latter +distinguishes removing Equality assembly from making net Equality/rebuild +baseline-equivalent and labels its implied point ratios as non-causal bounds. +At `--detail rulesets`, one driver panel per file unfolds exactly the Program and Equality cells from that table. Program children contain source rules' own five execution phases; Equality children contain every encoded-maintenance ruleset and one global native-Rebuild replacement row when its delta is nonzero. The @@ -476,7 +479,7 @@ requirements and which are only report presentation. | Scope-safe accounting | Preserve roles and accumulated time across push/pop, and subtract nested process/ruleset intervals from command timers | Otherwise nested schedules, checks, and rulesets are double-counted | One exclusive-subtraction boundary around commands and lowering; Residual verifies closure | | Wire format | Persist exact measurements without fixing the set of diagnostic counters | A five-field record could answer today's headline but would lose the Assembly/Search evidence that chose different optimization PRs | One sorted open list of segmented `path -> ns` leaves; no parent totals and no separate per-ruleset record | | Analysis | Align independent endpoint samples, derive Residual, and unfold Program and Equality into named children | Source own work, encoded maintenance, and native Rebuild must remain separate to keep every sign truthful | One generic exact-path sample map; the two parent groups equal the decomposition directly | -| Presentation and validation | Render one scan-first mechanism table and one compact driver panel per file; test additive closure at both hierarchy levels | Only the five buckets hides which ruleset carries Assembly/Search, while role totals falsely attach global Rebuild to source rules | Two mechanism parents, source top five plus exact Other, all maintenance children, and one native-Rebuild child | +| Presentation and validation | Render one scan-first mechanism table, one suite counterfactual table, and one compact driver panel per file; test additive closure at both hierarchy levels | Only the five buckets hides which ruleset carries Assembly/Search, while raw deltas still require manual arithmetic to answer optimization-ceiling questions and role totals falsely attach global Rebuild to source rules | Static delta-reset scenarios plus two mechanism parents, source top five plus exact Other, all maintenance children, and one native-Rebuild child | The minimization pass removed or avoided the main sources of accidental complexity: diff --git a/tests/__snapshots__/test_report_rendering.ambr b/tests/__snapshots__/test_report_rendering.ambr index 89347f2d..c73d43dc 100644 --- a/tests/__snapshots__/test_report_rendering.ambr +++ b/tests/__snapshots__/test_report_rendering.ambr @@ -52,6 +52,21 @@ *Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* + ### Optimization ceilings + + | Hypothetical change | Time removed | Remaining wall Δ | Implied ratio | + | --- | ---: | ---: | ---: | + | Remove added typechecking time | 0 ns | +200 ms | 1.07x | + | Remove added frontend/install time | 0 ns | +200 ms | 1.07x | + | Remove added typechecking + frontend time | 0 ns | +200 ms | 1.07x | + | Remove added Equality assembly time | 0 ns | +200 ms | 1.07x | + | Remove added net Equality/rebuild time | 0 ns | +200 ms | 1.07x | + | Remove added source-rule execution time | +116 ms | +84.0 ms | 1.03x | + | Remove every added non-program mechanism | 0 ns | +200 ms | 1.07x | + | Remove every recorded added mechanism | +116 ms | +84.0 ms | 1.03x | + + *Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is never removed.* + ## Ruleset drivers *Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those cells and alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, and Merge; Equality children contain every encoded maintenance ruleset plus one global Native rebuild replaced row. ↳ marks children in every format. Zero children are hidden. Source children are ranked by absolute own-work Δ (top 5 plus an exact per-group Other); every nonzero maintenance child is shown. Important phases include every \|phase Δ\| ≥ max(1 ms, 10% of \|row Δ\|), always include the dominant phase (◆), and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases.* @@ -205,6 +220,24 @@ type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative. + Optimization ceilings + + Hypothetical change Time removed Remaining wall Δ Implied ratio + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Remove added typechecking time 0 ns +1550 ms 1.11x + Remove added frontend/install time 0 ns +1550 ms 1.11x + Remove added typechecking + frontend time 0 ns +1550 ms 1.11x + Remove added Equality assembly time 0 ns +1550 ms 1.11x + Remove added net Equality/rebuild time 0 ns +1550 ms 1.11x + Remove added source-rule execution time +165 ms +1385 ms 1.10x + Remove every added non-program mechanism 0 ns +1550 ms 1.11x + Remove every recorded added mechanism +165 ms +1385 ms 1.10x + + Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured + mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net + Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not + implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is + never removed. ─────────────────────────────────────────────────── Per-file results ─────────────────────────────────────────────────── Wall time diff --git a/tests/test_report_analysis.py b/tests/test_report_analysis.py index 7e91e522..9d4ce99a 100644 --- a/tests/test_report_analysis.py +++ b/tests/test_report_analysis.py @@ -371,6 +371,84 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp assert native_rebuild.delta == equality.delta +def test_optimization_ceilings_reset_suite_deltas_without_claiming_implementation_speedups( + tmp_path: Path, +) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + candidate_timing = make_timing_summary( + make_ruleset_timing( + assembly_ns=31, + search_ns=37, + apply_ns=41, + execution_ns=43, + merge_ns=47, + rebuild_ns=53, + ), + make_ruleset_timing( + name="@rebuilding", + role="equality", + assembly_ns=61, + search_ns=0, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=0, + ), + typecheck_ns=13, + frontend_parse_ns=11, + frontend_other_ns=17, + frontend_install_ns=19, + commands_actions_ns=23, + commands_check_ns=7, + commands_other_ns=29, + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + wall_sec=0.000001, + timing_summary=make_timing_summary( + make_ruleset_timing( + assembly_ns=0, + search_ns=0, + apply_ns=0, + execution_ns=0, + merge_ns=0, + rebuild_ns=0, + ) + ), + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + wall_sec=0.0000015, + timing_summary=candidate_timing, + ), + ) + + ceilings = analyze_pair(ReportStore(report), comparison, "phases").ceilings + + assert [row.scenario for row in ceilings] == [ + "typecheck", + "frontend", + "frontend_and_typecheck", + "equality_assembly", + "equality", + "program", + "non_program", + "all_recorded", + ] + assert [row.reset_delta_ns for row in ceilings] == pytest.approx([13, 47, 60, 61, 114, 199, 233, 432]) + assert [row.remaining_delta_ns for row in ceilings] == pytest.approx([487, 453, 440, 439, 386, 301, 267, 68]) + assert [row.counterfactual_ratio for row in ceilings] == pytest.approx( + [1.487, 1.453, 1.440, 1.439, 1.386, 1.301, 1.267, 1.068] + ) + + @pytest.mark.parametrize( ("path", "message"), ((["residual", "stored"], "residual is derived"), (["mystery", "work"], "unknown timing responsibility")), diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index c016f578..1fa29403 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -229,14 +229,14 @@ def test_all_rich_tables_use_one_compact_style(tmp_path: Path) -> None: assert all(table.box is box.SIMPLE_HEAVY and not table.show_lines for table in tables) -def test_phase_detail_is_one_additive_decomposition_table(tmp_path: Path) -> None: +def test_phase_detail_adds_compact_suite_optimization_ceilings(tmp_path: Path) -> None: report_path, comparison = _pair_case(tmp_path) catalog = build_report_catalog(ReportStore(report_path), comparison, "phases") section = next(section for section in catalog.sections if section.id == "phases") tables = tuple(block for block in section.blocks if isinstance(block, ReportTable)) - assert len(tables) == 1 - table = tables[0] + assert len(tables) == 2 + table, ceilings = tables assert tuple(column.id for column in table.columns) == ( "file", "wall_delta", @@ -261,6 +261,26 @@ def test_phase_detail_is_one_additive_decomposition_table(tmp_path: Path) -> Non assert table.rows[1].cells[1].tone == "positive" assert table.rows[1].cells[4].tone == "emphasis" assert table.rows[1].cells[7].tone == "positive" + assert ceilings.title == "Optimization ceilings" + assert tuple(column.id for column in ceilings.columns) == ( + "scenario", + "reset_delta", + "remaining_delta", + "counterfactual_ratio", + ) + assert [row.cells[0].display for row in ceilings.rows] == [ + "Remove added typechecking time", + "Remove added frontend/install time", + "Remove added typechecking + frontend time", + "Remove added Equality assembly time", + "Remove added net Equality/rebuild time", + "Remove added source-rule execution time", + "Remove every added non-program mechanism", + "Remove every recorded added mechanism", + ] + assert all(row.cells[3].display.endswith("x") for row in ceilings.rows) + assert ceilings.caption is not None and "accounting bounds, not implementation predictions" in ceilings.caption + assert "Residual is never removed" in ceilings.caption def test_ruleset_detail_unfolds_program_and_equality_with_explicit_children(tmp_path: Path) -> None: From 424d05b06f29d5b3b97ef2c6c4ef197a4f3c40f5 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Wed, 12 Aug 2026 23:47:30 -0400 Subject: [PATCH 3/9] Document term encoding overhead --- README.md | 20 +- term-encoding-overhead-benchmark.md | 260 +++++++++ term-encoding-overhead-breakdown.md | 806 +++++++++------------------- 3 files changed, 511 insertions(+), 575 deletions(-) create mode 100644 term-encoding-overhead-benchmark.md diff --git a/README.md b/README.md index c7665a56..7b7aa57c 100644 --- a/README.md +++ b/README.md @@ -393,13 +393,13 @@ offset. `!` on a Residual cell means at least one endpoint's mean recorded total exceeded its wall time. A compact suite-level `Optimization ceilings` table then resets selected -candidate-minus-baseline deltas to zero and reports the remaining wall-time -change and implied point ratio. It distinguishes removing only Equality -ruleset assembly from making the entire net Equality/rebuild responsibility -match the baseline. These rows are optimistic additive accounting bounds, not -predictions: they hold every other measured mean fixed, omit confidence -intervals, and cannot model interactions between optimizations. Residual is -never treated as removable work. +positive candidate-minus-baseline deltas to zero and reports the remaining +wall-time change and implied point ratio. Candidate-side speedups are retained. +It distinguishes removing only Equality ruleset assembly from making the +entire net Equality/rebuild responsibility match the baseline. These rows are +optimistic additive accounting bounds, not predictions: they hold every other +measured mean fixed, omit confidence intervals, and cannot model interactions +between optimizations. Residual is never treated as removable work. At `--detail rulesets`, one compact driver table appears per file. Its `Program rules — own work` and `Equality/rebuild — net` parent rows exactly @@ -619,9 +619,9 @@ shown. No median or geometric mean is mixed into this minimal headline. A timed-out, failed, or otherwise incomplete selected result invalidates the suite result that depends on it. Valid per-file tail comparisons remain useful -when an unrelated file is incomplete. Mechanism contributions and individual -ruleset component deltas are descriptive diagnostics; ruleset totals receive -confidence intervals. +when an unrelated file is incomplete. Mechanism contributions, optimization +ceilings, and ruleset totals or component deltas are descriptive diagnostics; +only endpoint estimates and ratios receive confidence intervals. The `<2x` proof goal is established only when the upper bound of the suite wall ratio's 95% confidence interval is below `2x` for a proofs-versus-off diff --git a/term-encoding-overhead-benchmark.md b/term-encoding-overhead-benchmark.md new file mode 100644 index 00000000..e1984b5b --- /dev/null +++ b/term-encoding-overhead-benchmark.md @@ -0,0 +1,260 @@ +# Benchmark Report + +## Comparison + +| Role | Target | Git | Treatment | +| --- | --- | --- | --- | +| Baseline | d60202f64424 | d60202f64424 | off | +| Candidate | d60202f64424 | d60202f64424 | term | + +*10 file(s): math-microbenchmark-rational.egg, eggcc-2mm-pass1.egg, pointer-analysis-initdb.egg (facts: /Users/saul/p/wt/egglog-encoding/term-encoding-always-on/egglog/tests/pointer-analysis-initdb), hardboiled_conv1d_32.egg, luminal-llama.egg, herbie.egg, misaal-hvx-dot-product.egg, churchroad-wide-multiply.egg, dialegg-nmm40.egg, speq-preserved-reference-suite.egg · 6 round(s) per endpoint/file · 120 s timeout per run · Report: /private/tmp/term-encoding-d60202f.jsonl* + +## Summary — d60202f64424 term vs d60202f64424 off + +| Metric | Scope | File(s) | Ratio (95% CI) | Result | +| --- | --- | --- | ---: | --- | +| Wall time | Suite total | 10 files | 1.61–1.63x | slower | +| Wall time | Lowest-ratio file | churchroad-wide-multiply.egg | 0.705–0.715x | faster | +| Wall time | Highest-ratio file | speq-preserved-reference-suite.egg | 3.56–3.61x | slower | +| Peak RSS | Lowest-ratio file | churchroad-wide-multiply.egg | 1.10–1.11x | higher RSS | +| Peak RSS | Highest-ratio file | pointer-analysis-initdb.egg | 3.59–3.65x | higher RSS | + +*Ratios are candidate / baseline; below 1 is lower and above 1 is higher.* + +## Per-file results + +### Wall time + +| File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | +| --- | ---: | ---: | ---: | --- | +| math-microbenchmark-rational.egg | 393–412 ms | 825–836 ms | 2.01–2.11x | slower | +| eggcc-2mm-pass1.egg | 799–812 ms | 1.16–1.19 s | 1.43–1.48x | slower | +| pointer-analysis-initdb.egg | 57.1–58.4 ms | 132–137 ms | 2.27–2.37x | slower | +| hardboiled_conv1d_32.egg | 110–112 ms | 210–215 ms | 1.88–1.94x | slower | +| luminal-llama.egg | 355–363 ms | 1.23–1.23 s | 3.38–3.46x | slower | +| herbie.egg | 51.7–52.4 ms | 102–105 ms | 1.96–2.02x | slower | +| misaal-hvx-dot-product.egg | 33.2–33.9 ms | 99.3–103 ms | 2.95–3.07x | slower | +| churchroad-wide-multiply.egg | 1.00–1.02 s | 715–717 ms | 0.705–0.715x | faster | +| dialegg-nmm40.egg | 158–160 ms | 261–263 ms | 1.64–1.66x | slower | +| speq-preserved-reference-suite.egg | 45.7–46.0 ms | 163–165 ms | 3.56–3.61x | slower | + +### Peak RSS + +| File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | +| --- | ---: | ---: | ---: | --- | +| math-microbenchmark-rational.egg | 287.1–287.3 MiB | 494.0–494.6 MiB | 1.72–1.72x | higher RSS | +| eggcc-2mm-pass1.egg | 109.4–110.8 MiB | 248.2–250.2 MiB | 2.25–2.28x | higher RSS | +| pointer-analysis-initdb.egg | 41.7–42.4 MiB | 152.1–152.7 MiB | 3.59–3.65x | higher RSS | +| hardboiled_conv1d_32.egg | 41.6–41.9 MiB | 68.1–68.6 MiB | 1.63–1.64x | higher RSS | +| luminal-llama.egg | 118.1–119.4 MiB | 256.4–259.8 MiB | 2.15–2.19x | higher RSS | +| herbie.egg | 19.9–20.1 MiB | 34.0–34.2 MiB | 1.69–1.71x | higher RSS | +| misaal-hvx-dot-product.egg | 31.7–31.9 MiB | 65.5–65.9 MiB | 2.06–2.07x | higher RSS | +| churchroad-wide-multiply.egg | 20.5–20.6 MiB | 22.6–22.8 MiB | 1.10–1.11x | higher RSS | +| dialegg-nmm40.egg | 30.6–30.8 MiB | 97.8–98.0 MiB | 3.17–3.20x | higher RSS | +| speq-preserved-reference-suite.egg | 16.8–17.1 MiB | 35.2–35.7 MiB | 2.07–2.11x | higher RSS | + +## Slowdown decomposition + +| File | Wall Δ | Typecheck | Frontend | Program | Equality | Commands | Residual | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Suite total | +1889 ms | +18.2% +345 ms | +18.2% +344 ms | ◆ +29.8% +563 ms | +24.1% +455 ms | +7.22% +136 ms | +2.46% +46.4 ms | +| math-microbenchmark-rational.egg | +428 ms | +0.369% +1.58 ms | +0.387% +1.65 ms | +35.5% +152 ms | ◆ +60.6% +259 ms | +2.76% +11.8 ms | +0.397% +1.70 ms | +| eggcc-2mm-pass1.egg | +368 ms | +18.0% +66.2 ms | +18.9% +69.4 ms | ◆ +29.4% +108 ms | +22.7% +83.4 ms | +9.37% +34.5 ms | +1.68% +6.18 ms | +| pointer-analysis-initdb.egg | +76.4 ms | +2.88% +2.20 ms | +27.0% +20.7 ms | +14.9% +11.4 ms | ◆ +38.4% +29.3 ms | +4.36% +3.34 ms | +12.5% +9.53 ms | +| hardboiled_conv1d_32.egg | +101 ms | +24.6% +24.9 ms | +23.4% +23.7 ms | ◆ +32.9% +33.3 ms | +10.6% +10.7 ms | +5.65% +5.72 ms | +2.83% +2.86 ms | +| luminal-llama.egg | +868 ms | +21.0% +183 ms | +18.6% +162 ms | ◆ +46.5% +403 ms | +4.59% +39.8 ms | +7.43% +64.5 ms | +1.83% +15.9 ms | +| herbie.egg | +51.6 ms | +15.4% +7.96 ms | +17.7% +9.12 ms | ◆ +26.6% +13.7 ms | +22.4% +11.5 ms | +14.8% +7.62 ms | +3.23% +1.67 ms | +| misaal-hvx-dot-product.egg | +67.4 ms | +45.1% +30.4 ms | ◆ +45.5% +30.7 ms | +1.07% +0.724 ms | +2.81% +1.90 ms | +0.702% +0.473 ms | +4.86% +3.28 ms | +| churchroad-wide-multiply.egg | -292 ms | -1.90% +5.56 ms | -2.11% +6.18 ms | ◆ +105% -308 ms | -0.552% +1.61 ms | -0.150% +0.439 ms | -0.496% +1.45 ms | +| dialegg-nmm40.egg | +103 ms | +11.1% +11.4 ms | +9.85% +10.1 ms | ◆ +57.9% +59.5 ms | +16.3% +16.8 ms | +2.71% +2.78 ms | +2.11% +2.17 ms | +| speq-preserved-reference-suite.egg | +118 ms | +9.95% +11.8 ms | +9.13% +10.8 ms | ◆ +74.5% +88.2 ms | +0.660% +0.782 ms | +4.37% +5.18 ms | +1.41% +1.67 ms | + +*Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* + +### Optimization ceilings + +| Hypothetical change | Time removed | Remaining wall Δ | Implied ratio | +| --- | ---: | ---: | ---: | +| Remove added typechecking time | +345 ms | +1545 ms | 1.51x | +| Remove added frontend/install time | +344 ms | +1545 ms | 1.51x | +| Remove added typechecking + frontend time | +689 ms | +1201 ms | 1.40x | +| Remove added Equality assembly time | +299 ms | +1590 ms | 1.52x | +| Remove added net Equality/rebuild time | +455 ms | +1434 ms | 1.47x | +| Remove added source-rule execution time | +563 ms | +1327 ms | 1.44x | +| Remove every added non-program mechanism | +1.28 s | +609 ms | 1.20x | +| Remove every recorded added mechanism | +1.84 s | +46.4 ms | 1.02x | + +*Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is never removed.* + +## Ruleset drivers + +*Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those cells and alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, and Merge; Equality children contain every encoded maintenance ruleset plus one global Native rebuild replaced row. ↳ marks children in every format. Zero children are hidden. Source children are ranked by absolute own-work Δ (top 5 plus an exact per-group Other); every nonzero maintenance child is shown. Important phases include every \|phase Δ\| ≥ max(1 ms, 10% of \|row Δ\|), always include the dominant phase (◆), and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases.* + +### Ruleset drivers — math-microbenchmark-rational.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +152 ms | +35.5% | ◆ Apply +80.8 ms; Merge +69.9 ms; … | +| ↳ | +152 ms | | ◆ Apply +80.8 ms; Merge +69.9 ms; … | +| Equality/rebuild — net | +259 ms | +60.6% | ◆ Search +267 ms; Apply +57.6 ms; Merge +69.4 ms; Rebuild -135 ms; … | +| ↳ @rebuilding | +354 ms | | ◆ Search +233 ms; Apply +57.2 ms; Merge +62.8 ms; … | +| ↳ @parent | +41.0 ms | | ◆ Search +33.9 ms; Merge +6.60 ms; … | +| ↳ @rebuilding_cleanup | +855 ns | | ◆ Assembly +855 ns | +| ↳ @subsume_ruleset | +209 ns | | ◆ Assembly +209 ns | +| ↳ Native rebuild replaced | -135 ms | | ◆ Rebuild -135 ms | + +*Program + Equality account for +96.1% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* + +### Ruleset drivers — eggcc-2mm-pass1.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +108 ms | +29.4% | Assembly +11.1 ms; ◆ Search +54.5 ms; Apply +24.5 ms; Merge +17.7 ms; … | +| ↳ always-run | +82.2 ms | | Assembly +8.52 ms; ◆ Search +43.2 ms; Apply +18.1 ms; Merge +12.0 ms; … | +| ↳ type-analysis | +7.08 ms | | Search +1.53 ms; ◆ Apply +2.84 ms; Merge +2.39 ms; … | +| ↳ is-resolved | +4.99 ms | | ◆ Search +4.01 ms; … | +| ↳ terms | +3.52 ms | | ◆ Search +1.46 ms; … | +| ↳ terms-helpers | +3.48 ms | | ◆ Search +1.86 ms; … | +| ↳ Other (23 more source rulesets) | +7.06 ms | | Assembly +1.47 ms; ◆ Search +2.44 ms; Apply +1.35 ms; Merge +1.68 ms; … | +| Equality/rebuild — net | +83.4 ms | +22.7% | ◆ Assembly +245 ms; Search +55.5 ms; Execution +10.8 ms; Merge +10.6 ms; Rebuild -245 ms; … | +| ↳ @rebuilding | +277 ms | | ◆ Assembly +201 ms; Search +50.1 ms; … | +| ↳ @parent | +48.6 ms | | ◆ Assembly +41.6 ms; Search +5.31 ms; … | +| ↳ @subsume_ruleset | +2.78 ms | | ◆ Assembly +2.68 ms; … | +| ↳ @rebuilding_cleanup | +65.8 us | | ◆ Assembly +65.8 us | +| ↳ Native rebuild replaced | -245 ms | | ◆ Rebuild -245 ms | + +*Program + Equality account for +52.1% of this file's wall-time change. Source rules shown: 5/28 plus exact Other. Maintenance rules shown: 4/4.* + +### Ruleset drivers — pointer-analysis-initdb.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +11.4 ms | +14.9% | Apply +3.88 ms; ◆ Merge +7.28 ms; … | +| ↳ | +11.4 ms | | Apply +3.88 ms; ◆ Merge +7.28 ms; … | +| Equality/rebuild — net | +29.3 ms | +38.4% | ◆ Search +19.9 ms; Apply +3.84 ms; Merge +9.41 ms; Rebuild -4.56 ms; … | +| ↳ @rebuilding | +18.9 ms | | ◆ Search +12.7 ms; Apply +3.25 ms; Merge +2.63 ms; … | +| ↳ @parent | +15.0 ms | | ◆ Search +7.19 ms; Merge +6.78 ms; … | +| ↳ @rebuilding_cleanup | +1.29 us | | ◆ Assembly +1.29 us | +| ↳ @subsume_ruleset | +770 ns | | ◆ Assembly +770 ns | +| ↳ Native rebuild replaced | -4.56 ms | | ◆ Rebuild -4.56 ms | + +*Program + Equality account for +53.3% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* + +### Ruleset drivers — hardboiled_conv1d_32.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +33.3 ms | +32.9% | ◆ Search +23.6 ms; Apply +4.44 ms; … | +| ↳ | +31.8 ms | | ◆ Search +23.6 ms; Apply +3.85 ms; … | +| ↳ typechecking | +967 us | | ◆ Apply +590 us; … | +| ↳ amx | +497 us | | ◆ Assembly +497 us | +| Equality/rebuild — net | +10.7 ms | +10.6% | Assembly +4.74 ms; ◆ Search +5.76 ms; Apply +2.32 ms; Rebuild -3.73 ms; … | +| ↳ @rebuilding | +12.4 ms | | Assembly +3.87 ms; ◆ Search +4.82 ms; Apply +2.29 ms; … | +| ↳ @parent | +2.02 ms | | ◆ Search +943 us; … | +| ↳ @subsume_ruleset | +21.1 us | | ◆ Assembly +21.1 us | +| ↳ @rebuilding_cleanup | +2.68 us | | ◆ Assembly +2.68 us | +| ↳ Native rebuild replaced | -3.73 ms | | ◆ Rebuild -3.73 ms | + +*Program + Equality account for +43.5% of this file's wall-time change. Source rules shown: 3/3. Maintenance rules shown: 4/4.* + +### Ruleset drivers — luminal-llama.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +403 ms | +46.5% | Assembly +87.7 ms; ◆ Search +311 ms; … | +| ↳ fusion_grow | +169 ms | | ◆ Search +168 ms; … | +| ↳ fusion_pair | +147 ms | | ◆ Search +145 ms; … | +| ↳ direct_kernel | +36.3 ms | | ◆ Search +36.1 ms; … | +| ↳ matmul_backend | +29.9 ms | | ◆ Assembly +75.1 ms; Search -45.5 ms; … | +| ↳ fusion_merge | +14.7 ms | | ◆ Search +14.2 ms; … | +| ↳ Other (11 more source rulesets) | +7.04 ms | | ◆ Assembly +11.4 ms; Search -5.95 ms; … | +| Equality/rebuild — net | +39.8 ms | +4.59% | ◆ Assembly +42.3 ms; Rebuild -5.66 ms; … | +| ↳ @rebuilding | +44.7 ms | | ◆ Assembly +41.6 ms; … | +| ↳ @parent | +565 us | | ◆ Assembly +514 us; … | +| ↳ @subsume_ruleset | +217 us | | ◆ Assembly +133 us; … | +| ↳ @rebuilding_cleanup | +3.66 us | | ◆ Assembly +3.66 us | +| ↳ Native rebuild replaced | -5.66 ms | | ◆ Rebuild -5.66 ms | + +*Program + Equality account for +51.1% of this file's wall-time change. Source rules shown: 5/16 plus exact Other. Maintenance rules shown: 4/4.* + +### Ruleset drivers — herbie.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +13.7 ms | +26.6% | ◆ Assembly +8.06 ms; Apply +3.05 ms; Merge +1.98 ms; … | +| ↳ | +13.7 ms | | ◆ Assembly +8.06 ms; Apply +3.05 ms; Merge +1.98 ms; … | +| Equality/rebuild — net | +11.5 ms | +22.4% | ◆ Search +8.05 ms; Apply +1.88 ms; Merge +2.25 ms; Rebuild -1.98 ms; … | +| ↳ @rebuilding | +11.3 ms | | ◆ Search +6.58 ms; Apply +1.80 ms; Merge +1.76 ms; … | +| ↳ @parent | +2.20 ms | | ◆ Search +1.47 ms; … | +| ↳ @rebuilding_cleanup | +2.40 us | | ◆ Assembly +2.40 us | +| ↳ @subsume_ruleset | +1.55 us | | ◆ Assembly +1.55 us | +| ↳ Native rebuild replaced | -1.98 ms | | ◆ Rebuild -1.98 ms | + +*Program + Equality account for +48.9% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* + +### Ruleset drivers — misaal-hvx-dot-product.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +724 us | +1.07% | ◆ Assembly +623 us; … | +| ↳ | +724 us | | ◆ Assembly +623 us; … | +| Equality/rebuild — net | +1.90 ms | +2.81% | ◆ Assembly +1.49 ms; … | +| ↳ @rebuilding | +2.03 ms | | ◆ Assembly +1.48 ms; … | +| ↳ @parent | +48.1 us | | ◆ Search +24.3 us; … | +| ↳ @rebuilding_cleanup | +230 ns | | ◆ Assembly +230 ns | +| ↳ @subsume_ruleset | +90.2 ns | | ◆ Assembly +90.2 ns | +| ↳ Native rebuild replaced | -180 us | | ◆ Rebuild -180 us | + +*Program + Equality account for +3.89% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* + +### Ruleset drivers — churchroad-wide-multiply.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | -308 ms | +105% | ◆ Search -309 ms; … | +| ↳ mapping | -309 ms | | ◆ Search -309 ms; … | +| ↳ transform | +899 us | | ◆ Apply +641 us; … | +| ↳ typing | +699 us | | ◆ Apply +437 us; … | +| ↳ misc | +6.33 us | | ◆ Assembly +6.33 us | +| Equality/rebuild — net | +1.61 ms | -0.552% | ◆ Assembly +1.21 ms; … | +| ↳ @rebuilding | +1.43 ms | | ◆ Assembly +1.02 ms; … | +| ↳ @parent | +187 us | | ◆ Assembly +178 us; … | +| ↳ @rebuilding_cleanup | +1.31 us | | ◆ Assembly +1.31 us | +| ↳ @subsume_ruleset | +1.22 us | | ◆ Assembly +1.22 us | + +*Program + Equality account for +105% of this file's wall-time change. Source rules shown: 4/4. Maintenance rules shown: 4/4.* + +### Ruleset drivers — dialegg-nmm40.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +59.5 ms | +57.9% | ◆ Apply +40.4 ms; Merge +22.8 ms; … | +| ↳ rules | +59.5 ms | | ◆ Apply +40.4 ms; Merge +22.8 ms; … | +| Equality/rebuild — net | +16.8 ms | +16.3% | Assembly +1.73 ms; ◆ Search +15.4 ms; Apply +6.26 ms; Merge +4.05 ms; Rebuild -11.1 ms; … | +| ↳ @rebuilding | +26.9 ms | | ◆ Search +14.6 ms; Apply +6.25 ms; Merge +3.95 ms; … | +| ↳ @parent | +950 us | | ◆ Search +747 us; … | +| ↳ @rebuilding_cleanup | +701 ns | | ◆ Assembly +701 ns | +| ↳ @subsume_ruleset | +334 ns | | ◆ Assembly +334 ns | +| ↳ Native rebuild replaced | -11.1 ms | | ◆ Rebuild -11.1 ms | + +*Program + Equality account for +74.2% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* + +### Ruleset drivers — speq-preserved-reference-suite.egg + +| Driver | Δ | Wall share | Important phase changes | +| --- | ---: | ---: | --- | +| Program rules — own work | +88.2 ms | +74.5% | ◆ Assembly +71.8 ms; Search +9.67 ms; … | +| ↳ parseIR.transform-taco-spmv-csc | +27.8 ms | | ◆ Assembly +19.7 ms; Search +4.84 ms; Execution +3.23 ms; … | +| ↳ parseIR.transform-csparse-spmv-csc-nostruct | +27.4 ms | | ◆ Assembly +19.4 ms; Search +4.74 ms; Execution +3.20 ms; … | +| ↳ parseIR.transform-parboil-hist | +16.5 ms | | ◆ Assembly +16.3 ms; … | +| ↳ parseIR.transform-npb-is-hist | +16.3 ms | | ◆ Assembly +16.2 ms; … | +| ↳ parseIR.expand-parboil-hist | +49.8 us | | ◆ Assembly +25.3 us; … | +| ↳ Other (3 more source rulesets) | +117 us | | ◆ Assembly +73.8 us; … | +| Equality/rebuild — net | +782 us | +0.660% | ◆ Assembly +478 us; … | +| ↳ @rebuilding | +783 us | | ◆ Assembly +440 us; … | +| ↳ @parent | +53.3 us | | ◆ Assembly +35.5 us; … | +| ↳ @subsume_ruleset | +1.27 us | | ◆ Assembly +1.27 us | +| ↳ @rebuilding_cleanup | +1.26 us | | ◆ Assembly +1.26 us | +| ↳ Native rebuild replaced | -56.5 us | | ◆ Rebuild -56.5 us | + +*Program + Equality account for +75.1% of this file's wall-time change. Source rules shown: 5/8 plus exact Other. Maintenance rules shown: 4/4.* diff --git a/term-encoding-overhead-breakdown.md b/term-encoding-overhead-breakdown.md index 2df861f2..acabe141 100644 --- a/term-encoding-overhead-breakdown.md +++ b/term-encoding-overhead-breakdown.md @@ -1,580 +1,256 @@ # Where Current Term-Encoding Time Goes -Status: measured diagnostic on current `origin/main`, followed by the retained -V3 phase-timing implementation documented in the evidence ledger below. +## Result -Baseline commit: `5ead0a0cacf847a129294a870de13503f2d7f9c4` +Term encoding is `1.61–1.63x` slower across the current ten-workload suite, but +there is no single dominant cause. The suite mean adds 1.889 seconds over a +3.035-second `off` baseline: -This commit includes PR #61, which records direct ruleset runs in the overall -report. That fix is necessary, but it does not account for rule planning and -assembly or for the frontend pipeline outside ruleset execution. - -## Short answer - -There is no single dominant term-encoding cost across the benchmark suite. - -- Math is dominated by replacing native rebuilding with `@rebuilding` and - `@parent` plus the extra Apply/Merge work of the encoded representation. -- Pointer analysis is dominated by generating, desugaring, typechecking, and - installing the encoded program. Its actual rule execution is tiny. -- Hardboiled is split between the generated frontend and slower transformed - user rules. -- Luminal is dominated by two nearly equal costs: generated frontend work and - planning/searching transformed user rules. Encoded UF/rebuild maintenance is - only about 5.5% of its slowdown. -- Herbie is mixed across frontend, transformed user rules, encoded - maintenance, and command work. -- eggcc is also mixed. A previously hidden ruleset-assembly cost is important: - assembling `@rebuilding` and `@parent` costs about 253 ms, before their - Search/Apply/Merge timers begin. - -Therefore the current two-part model is incomplete: - -1. encoded UF and rebuild maintenance; -2. generating and re-typechecking an encoded program; - -There is a third material cost: - -3. changed physical rule shape, including per-invocation rule assembly, - query planning, and user-rule search. - -There is also smaller workload-dependent top-level command work. - -## Question and hypotheses - -Question: for each of the six representative workloads, which mechanisms -explain the wall-time increase from `off` to `term` on the same binary? - -The diagnostic distinguished these competing hypotheses: - -- H1: explicit encoded UF/rebuild maintenance dominates. -- H2: the second frontend and generated-program installation dominate. -- H3: source rules become physically different queries and spend more time in - rule assembly/planning/search even when generated maintenance is cheap. -- H4: top-level actions, input loading, checks, extraction, or scheduler-driver - work dominate outside the rulesets. - -The results support different hypotheses on different workloads. +| Mechanism | Mean delta | Share of slowdown | +| --- | ---: | ---: | +| Source-rule execution | +563 ms | 29.8% | +| Equality/rebuild, net | +455 ms | 24.1% | +| Typechecking | +345 ms | 18.2% | +| Other frontend/install | +344 ms | 18.2% | +| Commands | +136 ms | 7.22% | +| Residual | +46.4 ms | 2.46% | + +The full generated report is checked in as +[`term-encoding-overhead-benchmark.md`](term-encoding-overhead-benchmark.md). + +The important engineering conclusion is that a native or inline rebuild alone +cannot reach the 5–10% target. Making the entire Equality/rebuild bucket as +cheap as the baseline would reduce the suite point ratio only from `1.62x` to +`1.47x`. Removing all measured non-program overhead would still leave `1.20x` +because transformed source rules remain materially different. Reaching `1.10x` +would require removing about 84% of the current added time, including roughly +306 ms, or 54%, of the net Program bucket even after every positive non-program +delta disappeared. ## Measurement -The decisive report is: - -```text -/tmp/term-overhead-main-5ead0a0-instrumented-assembly-v1.jsonl -``` - -The instrumented binary SHA-256 is: - -```text -3c140fc59901ec0d448778dc511f23ed12498434344ebd74010fe4db75dcd48e -``` - -Command shape: +The branch first merged `origin/main` at +`46f69b70d0819b03da110e6e785f91c080d58556`. The measured executable state is +commit `d60202f64424`; the later report-only commit does not change that binary. -```text +```bash ./bench.py \ - --target . --treatment term \ - --compare-target . --compare-treatment off \ - --rounds 6 --timeout-sec 120 --force-run \ - --report /tmp/term-overhead-main-5ead0a0-instrumented-assembly-v1.jsonl -``` - -Both endpoints use the same release binary, workload bytes, fact-directory -bytes, and one execution thread. Runs alternate `off` and `term` for each file. -All 72 runs succeeded. - -The instrumentation measured disjoint intervals for: - -- initialization and source-file reading; -- source parsing, macros, typechecking, and other source resolution; -- encoding generation, including parsing emitted encoding text; -- generated desugaring and generated typechecking; -- installing functions and compiled rules; -- top-level actions, input, scheduler-driver work, and other commands; -- per-ruleset assembly, Search, Apply, unattributed execution, Merge, and - Rebuild. - -The report's additive residual is wall time minus all those intervals. It is -only 0.5% to 6.2% of each measured slowdown, which is a useful check that the -phase boundaries explain nearly all of the difference. - -An earlier stock-main report, without the temporary extra timers, is retained -at: - -```text -/tmp/term-overhead-main-5ead0a0-v1.jsonl + --detail rulesets \ + --treatment term \ + --force-run \ + --report /tmp/term-encoding-d60202f.jsonl \ + --format markdown ``` -Its ratios agree with the diagnostic run. A separate ten-round diagnostic run -at `/tmp/term-overhead-main-5ead0a0-instrumented-v1.jsonl` contains substantial -machine-contention outliers and is retained rather than filtered. During the -campaign an unrelated long-running process was executing -`/tmp/churchroad-wide-multiply.egg`. The decisive six-round paired run was -stable despite that background process; its tight paired confidence intervals -support the relative decomposition, while its absolute milliseconds remain -machine-specific. - -## Additive slowdown decomposition +This collected six fresh `off` and six fresh `term` observations for each of +the ten default workloads: 120/120 runs succeeded. Both endpoints used the +same release binary, workload bytes, fact-directory bytes, timeout, and one +execution thread. Endpoint samples are treated as independent because the +JSONL intentionally stores no round-pair identity. -All numbers are paired mean `term - off` wall milliseconds over six rounds. -Percentages are shares of that file's wall-time increase. +The actual wall-time ratios were: -| Workload | Off | Term | Slowdown | Frontend | Transformed user rules | UF/rebuild substitution | Command work | Residual | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Math | 376.6 | 839.8 | 463.2 | 3.4 (0.7%) | 166.8 (36.0%) | 289.2 (62.4%) | 1.1 (0.2%) | 2.7 (0.6%) | -| eggcc | 839.5 | 1235.2 | 395.7 | 142.3 (36.0%) | 118.8 (30.0%) | 87.4 (22.1%) | 37.6 (9.5%) | 9.7 (2.4%) | -| Pointer | 8.0 | 15.5 | 7.6 | 6.4 (83.9%) | 0.2 (2.2%) | 0.6 (7.3%) | 0.0 (0.5%) | 0.5 (6.2%) | -| Hardboiled | 115.6 | 222.1 | 106.5 | 49.4 (46.4%) | 35.2 (33.1%) | 11.5 (10.8%) | 6.9 (6.5%) | 3.4 (3.2%) | -| Luminal | 372.5 | 1291.0 | 918.5 | 357.8 (39.0%) | 418.1 (45.5%) | 50.5 (5.5%) | 68.2 (7.4%) | 24.0 (2.6%) | -| Herbie | 54.3 | 109.5 | 55.2 | 17.5 (31.8%) | 14.7 (26.7%) | 12.0 (21.7%) | 9.4 (17.1%) | 1.5 (2.7%) | - -The paired 95% confidence intervals for the total slowdowns are: - -| Workload | Paired slowdown 95% CI | +| Workload | `term / off` (95% CI) | | --- | ---: | -| Math | 451.4–475.1 ms | -| eggcc | 384.8–406.7 ms | -| Pointer | 7.2–7.9 ms | -| Hardboiled | 103.4–109.6 ms | -| Luminal | 889.1–947.9 ms | -| Herbie | 53.7–56.7 ms | - -### Category definitions - -The table is exactly additive. - -- Frontend includes initialization, source-file reading, source parsing, - macros, source typechecking, other source resolution, encoding generation, - generated desugaring, generated typechecking, and installing functions and - rules. -- Transformed user rules includes Assembly, Search, Apply, Execution overhead, - and Merge for rulesets whose names do not begin with `@`. Rebuild is excluded - from this column. -- UF/rebuild substitution includes every phase of generated `@...` maintenance - rules plus the `term - off` difference in non-`@` Rebuild. This subtracts the - native rebuilding that term mode replaces, rather than presenting generated - maintenance as if native rebuilding were free. -- Command work includes top-level actions, input, schedule interpretation after - subtracting all recorded ruleset phases, and checks/extraction/other commands. -- Residual is process wall time not claimed by any measured phase. - -## What the rule timers say - -### Math: encoded maintenance really is the main problem - -Term mode spends 443.0 ms in generated `@...` maintenance: - -- 300.7 ms Search; -- 63.3 ms Apply; -- 78.5 ms Merge; -- less than 0.5 ms assembly and unattributed work. - -Off mode spends 153.7 ms in native Rebuild. Replacing that native work therefore -costs a net 289.2 ms. The transformed default ruleset adds another 88.0 ms Apply -and 76.8 ms Merge. Generated typechecking is only 1.7 ms. - -This is the workload where optimizing or native-backing the encoded UF/rebuild -machinery is directly decisive. - -### eggcc: rule assembly was hiding outside the report - -Term mode's generated maintenance costs 339.8 ms, but 253.0 ms of that is -ruleset assembly before Search starts: - -- `@rebuilding` assembly: 206.8 ms; -- `@parent` assembly: 43.4 ms; -- other generated maintenance assembly: about 2.7 ms. - -Off mode spends 252.5 ms in native Rebuild, leaving a net UF/rebuild -substitution cost of 87.4 ms. - -The frontend adds 142.3 ms: - -- encoding generation: 37.0 ms; -- generated desugaring: 13.1 ms; -- generated typechecking: 71.0 ms; -- installation: 23.6 ms; -- small offsets in the unchanged source frontend: about -2.4 ms. - -Transformed user rules add 118.8 ms, including 57.6 ms Search, 26.6 ms Apply, -21.1 ms Merge, and 13.0 ms assembly. Schedule-driver work adds another 36.6 ms. - -So neither "UF" nor "re-typechecking" alone explains eggcc. Avoiding repeated -assembly of the large generated maintenance rulesets is also a first-class -opportunity. - -### Pointer: almost entirely the second frontend - -The total slowdown is only 7.6 ms, but 6.4 ms is frontend work: - -- encoding generation: 2.3 ms; -- generated desugaring: 0.5 ms; -- generated typechecking: 2.3 ms; -- installation: 1.2 ms. - -User-rule execution adds about 0.2 ms and the net maintenance substitution adds -about 0.6 ms. This benchmark primarily measures fixed per-program encoding -cost, not UF execution. - -### Hardboiled: frontend first, changed user search second - -The frontend contributes 49.4 ms, led by 25.8 ms generated typechecking and -12.1 ms encoding generation. Transformed user rules add 35.2 ms, including -24.6 ms Search. Net encoded maintenance is 11.5 ms. - -### Luminal: not a UF bottleneck - -The 918.5 ms slowdown divides primarily into: - -- 357.8 ms frontend; -- 418.1 ms transformed user rules; -- 68.2 ms command work; -- only 50.5 ms net UF/rebuild substitution. - -The frontend includes 192.0 ms generated typechecking, 88.2 ms encoding, -43.7 ms installation, and 25.4 ms generated desugaring. Top-level encoded -actions add 65.0 ms. - -Within transformed user rules, Search adds 322.3 ms and assembly adds 90.9 ms. -This agrees with the corrected ruleset report: `fusion_grow` and `fusion_pair` -search much slower under the wider encoded relation/query shapes. Improving -`@UF` alone cannot materially close Luminal's gap. - -### Herbie: no single dominant bucket - -Frontend contributes 17.5 ms, transformed user rules 14.7 ms, net maintenance -12.0 ms, and command work 9.4 ms. The command bucket includes about 7.7 ms in -checks/extraction/other commands across the fixture's repeated push/pop scopes. - -## Is rule-execution difference enough to measure encoding overhead? - -It is necessary, but not sufficient. - -After PR #61, the existing report can correctly expose: - -- generated maintenance rulesets such as `@rebuilding` and `@parent`; -- Search, Apply, Execution overhead, Merge, and Rebuild; -- changes in source-named rulesets after encoding. - -That is enough to identify Math's main bottleneck and Luminal's changed Search -shape. It is not enough for Pointer, eggcc, Hardboiled, or Herbie because it -still leaves these costs in one residual: - -- source versus generated typechecking; -- encoding generation and parsing of emitted text; -- generated desugaring; -- function/rule installation; -- ruleset assembly and per-invocation plan construction; -- top-level actions and command work. - -In particular, looking only at the five currently reported ruleset phases would -misclassify eggcc's approximately 253 ms generated-maintenance assembly cost as -frontend or generic outside overhead. - -## Minimal permanent reporting change - -A narrow reporting PR should add measurement before optimization PRs. - -1. Add `Assembly` before `Search` to each per-ruleset timing record. It should - include first-use cached-plan construction and rebuilding the executable - ruleset for each invocation. -2. Add disjoint outside-ruleset leaves under two explicit parents: - - Lowering / Parse, total Typecheck, and Other; - - Commands / Install, Actions/input, and Other/schedules; - - residual derived per observation from process wall time. -3. Charge source typechecking performed in the separate original-typechecker - e-graph to the outer total Typecheck leaf. Keeping one shared Typecheck leaf - makes the off-versus-encoded delta directly show the checking added by the - encoding, without storing mode-specific fields. -4. Render an additive per-file slowdown table like the one above, in addition - to detailed ruleset rows. Do not infer "frontend" from the wall-time - residual after collection. -5. Preserve same-binary comparisons and one-thread execution for additive - Search/Apply timing. - -The diagnostic patch used here added 196 lines across five Rust files. That is -too broad and ad hoc to retain as production code, but it validates the phase -boundaries for a smaller, reviewed implementation. A four-round clean-versus- -instrumented timer-tax comparison is retained at -`/tmp/term-overhead-timer-tax-off-v1.jsonl`; machine contention made its wall -CI inconclusive, while RSS was indistinguishable on nearly all workloads. A -production PR should include a cleaner timer-tax check. - -## Consequences for the unification roadmap - -The measurements argue against a single "replace encoded UF" campaign. - -- Preserve the one-relational-UF destination, but treat it as the Math-focused - and maintenance-focused track. -- Typed emission that removes generated parsing/desugaring/typechecking is the - direct track for Pointer and a large part of eggcc, Hardboiled, Luminal, and - Herbie. -- Identity/view fusion that restores narrow source query shapes is at least as - important as UF fusion for Luminal. -- Cache or eliminate repeated assembly of generated maintenance rulesets; - otherwise eggcc can spend more time preparing `@rebuilding` than executing - its Search/Apply/Merge phases. -- Keep top-level encoded action lowering visible. It is about 65 ms of - Luminal's slowdown even after the frontend is separated. - -The performance target should be evaluated per workload as well as in suite -aggregate. A change that fixes Math's `@rebuilding` cost can leave Pointer and -Luminal almost untouched, while a typed-frontend change can make Pointer much -faster and barely move Math. - -## P0b implementation evidence ledger - -Status: complete on `codex/term-encoding-always-on` from -`5ead0a0cacf847a129294a870de13503f2d7f9c4`. - -Smallest falsifiable contract: a synthetic V3 observation assigns distinct -nanosecond values to every exclusive leaf under Lowering, Commands, and -Ruleset. The report must display every leaf once, and the leaves plus the -derived residual must reconstruct external wall time. A real CLI fixture must -also show nonzero source Parse, Typecheck, Commands / Install, Commands / -Actions, and Ruleset / Assembly values. - -Current hypothesis: most of the previously unexplained proof/term-encoding -slowdown can be localized without splitting source and generated passes. Total -Typecheck is enough to answer how much extra typechecking the encoded mode -adds, provided typechecking done by the cloned source checker is charged to the -outer e-graph. Ruleset / Assembly must include first-use cached-plan creation -and per-invocation executable-ruleset materialization, while core execution -setup belongs to Ruleset / Execution overhead. - -Falsifiers: - -- nested command or schedule time appears both in Commands and Ruleset; -- source typechecking in encoded modes falls into Lowering / Other; -- generated parsing is omitted from Lowering / Parse; -- internal rebuild-rule assembly is recorded again outside Ruleset / Rebuild; -- the synthetic leaves plus residual do not equal wall time; -- timer instrumentation causes a material wall-time regression in an - instrumented-versus-clean same-treatment comparison. - -Evidence to retain: focused Rust producer/CLI tests, focused Python -schema-analysis-rendering tests, `make check`, `make benchmark-smoke`, the Rich -and Markdown width matrix, one real off-versus-proofs phase report, and a -timer-tax comparison. Failed hypotheses and inconclusive timing intervals stay -recorded rather than being discarded. - -Implementation result: - -- The producer and consumer contract is now V3. The initial tests failed on the - missing nested process schema, Assembly field, and analysis types, then pass - with all thirteen leaves reconstructing synthetic wall time exactly. -- `/tmp/term-overhead-off-proofs-v3.jsonl` contains four fresh rounds for all - six default workloads. The suite proof/off wall ratio is 3.04–3.16x. Its - phase tables leave small residual shares on the substantive workloads and - make the previously hidden Typecheck, Install, Actions/input, and Assembly - changes explicit. -- `/tmp/term-overhead-timer-tax-v3.jsonl` compares ten rounds of the - instrumented off mode with a temporary V3-compatible `5ead0a0` control. The - suite means are 1.7591 s versus 1.7408 s, a 1.0105x point ratio; the displayed - 95% interval rounds to 1.00–1.02x. This rejects a 5–10% suite-level timer tax, - though it does detect a small roughly 1% effect. -- `/tmp/term-overhead-timer-tax-proofs-v3.jsonl` repeats the control comparison - for proofs over four rounds; the suite interval is 0.873–1.07x and therefore - inconclusive, with every per-file wall interval including 1. -- Focused and six-file Rich reports render successfully at widths 80, 119, 120, - 160, and 200. Widths 80 and 119 emit exactly one detailed-report warning; - wider reports emit none. Markdown output is byte-identical across all five - widths for each scope. -- `make check` and `make benchmark-smoke` both pass in the implementation - worktree. - -## Flat mechanism-ledger follow-up - -Status: supersedes the fixed nested V3 transport above while retaining its -timer sites and the ruleset Assembly measurement. - -The persisted timing summary is now one sorted list of exclusive leaves: - -```json -{"schema_version":3,"timings":[{"path":["program","search","fusion_grow"],"ns":123}]} -``` - -The first path segment is the additive responsibility shown in the report; -deeper segments retain diagnostic resolution. The stable responsibilities are -Typecheck, Frontend, Program, Equality, and Commands. Residual remains derived -as process wall time minus the sum of every leaf. No parent total is stored. -Ruleset names are separate path segments, so names containing `/` cannot be -misparsed. - -Rulesets receive an explicit timing role when declared. Program rules write -Assembly, Search, Apply, Execution, and Merge under `program`; their native -Rebuild tail writes under `equality/rebuild`. Encoded maintenance rules write -all phases under `equality`. Thus the net cost of replacing native rebuilding -with relational maintenance is an ordinary candidate-minus-baseline Equality -difference, not a reporting-time credit calculation or an `@`-prefix guess. - -Checks have their own `commands/check` leaf. The transient backend query and -the surrounding compilation/validation overhead are charged there in both off -and encoded modes. The motivating claim that term-mode checks themselves were -showing up as the default ruleset was falsified by check-only CLI probes: the -old report recorded no transient check ruleset in either mode. Hardboiled has a -real default `(run)`. Keeping the explicit check leaf still removes the -ambiguity and prevents future routing asymmetry. - -One boundary remains intentionally command-scoped: a top-level action such as -`(union ...)` can trigger `flush_updates` and native rebuilding, but its -transient backend report is not a named ruleset run. That entire interval is -therefore recorded under `commands/actions`, not `equality/rebuild`. - -The fresh six-round report confirms the separation on real fixtures. -Hardboiled records `commands/check` means of 1.697 ms off and 3.299 ms term, -while its independent default-ruleset Search means are 60.349 ms and 84.870 -ms. Herbie records 0.175 ms and 0.241 ms for checks. Check evaluation is -therefore visible without being mistaken for transformed program-rule Search. - -At `--detail phases`, presentation starts with an additive -slowdown-decomposition table with a Suite row and one row per file, followed -by a compact suite-level table of optimistic accounting ceilings. The latter -distinguishes removing Equality assembly from making net Equality/rebuild -baseline-equivalent and labels its implied point ratios as non-causal bounds. -At `--detail rulesets`, one driver panel per file unfolds exactly the Program and -Equality cells from that table. Program children contain source rules' own five -execution phases; Equality children contain every encoded-maintenance ruleset -and one global native-Rebuild replacement row when its delta is nonzero. The -two parent phase summaries retain the unique diagnostic question from the -removed global rollup: whether Program or Equality cost is Assembly, Search, -Rebuild, or another execution phase. Up to five source children plus an exact -per-group Other are shown, while the small fixed set of nonzero maintenance -children is shown in full. Native Rebuild is never attributed to whichever -source ruleset happened to trigger it. - -### Readability rationale - -The headline table follows a task-first rather than decorative color design. -A [controlled IEEE VIS table-reading -study](https://ieeexplore.ieeevis.org/year/2024/program/paper_v-full-1288.html) -found that visual aids are task dependent: color and bar encodings help some -extrema tasks, while row striping performs better for some complex comparison -tasks. [W3C table guidance](https://www.w3.org/WAI/tutorials/tables/tips/) -likewise recommends row orientation aids with sufficient contrast, and -[WCAG's use-of-color guidance](https://www.w3.org/WAI/WCAG20/Understanding/use-of-color) -requires that color not be the only signal. - -Accordingly, percent share comes first for vertical comparison, every report -table uses the same compact header-rule style, and `◆` identifies the largest -absolute mechanism share. Rich and interactive reports also bold that dominant -cell and dim contributions below 5%. Expected added overhead is neutral; green -is reserved for improvements, while yellow and red are reserved for measurement -warnings and errors. Signed values and the `◆` marker keep the meaning -independent of color. The contributor panels stay textual rather than adding in-cell bars: -the primary task is finding a dominant role, ruleset, and phase, and bars would -add another renderer-specific encoding to an already dense diagnostic. - -### Complexity audit and minimization - -The retained complexity falls into six distinct responsibilities. Keeping -them separate makes it possible to decide which parts are measurement -requirements and which are only report presentation. - -| Layer | Added responsibility | Why it remains | Simplification retained | -| --- | --- | --- | --- | -| Engine phase boundaries | Measure seven process leaves and six exclusive ruleset phases, including Assembly | Without these boundaries, Pointer frontend time and eggcc plan construction return to an undifferentiated residual | Static path slices and one duration map; no mode-specific timer structs | -| Semantic routing | Route source rules, relational equality maintenance, native Rebuild, and transient checks consistently | Equality is a responsibility implemented differently by the two treatments, so name-prefix inference or a report-time credit gives the wrong abstraction | One two-variant role enum; checks use one symmetric `commands/check` path | -| Scope-safe accounting | Preserve roles and accumulated time across push/pop, and subtract nested process/ruleset intervals from command timers | Otherwise nested schedules, checks, and rulesets are double-counted | One exclusive-subtraction boundary around commands and lowering; Residual verifies closure | -| Wire format | Persist exact measurements without fixing the set of diagnostic counters | A five-field record could answer today's headline but would lose the Assembly/Search evidence that chose different optimization PRs | One sorted open list of segmented `path -> ns` leaves; no parent totals and no separate per-ruleset record | -| Analysis | Align independent endpoint samples, derive Residual, and unfold Program and Equality into named children | Source own work, encoded maintenance, and native Rebuild must remain separate to keep every sign truthful | One generic exact-path sample map; the two parent groups equal the decomposition directly | -| Presentation and validation | Render one scan-first mechanism table, one suite counterfactual table, and one compact driver panel per file; test additive closure at both hierarchy levels | Only the five buckets hides which ruleset carries Assembly/Search, while raw deltas still require manual arithmetic to answer optimization-ceiling questions and role totals falsely attach global Rebuild to source rules | Static delta-reset scenarios plus two mechanism parents, source top five plus exact Other, all maintenance children, and one native-Rebuild child | - -The minimization pass removed or avoided the main sources of accidental -complexity: - -- fixed nested timing structs and a second per-ruleset wire schema were - replaced by the one open path ledger; -- the native-Rebuild "credit" disappeared because both equality - implementations are recorded under `equality` before analysis; -- `@`-prefix classification disappeared in favor of declaration-time roles; -- mechanism and ruleset-driver reports now share one aligned sample map; -- the global depth-two rollup and ten-column ruleset tables became one compact - four-column panel per file whose two parent rows directly match Program and - Equality, with truthful per-group children and an exact source remainder; -- report invariants use ordinary exceptions, so `python -O` cannot remove - cache-safety checks; -- the visual treatment uses existing table primitives rather than adding bar - geometry or renderer-specific calculations. - -Two tempting reductions would make the design simpler only on paper. -Collapsing persistence to the five headline buckets would make the measured -eggcc Assembly and Luminal Search costs inseparable. Inferring maintenance -from generated names would remove the role enum but make semantics depend on a -printer convention. Neither is retained. - -The one plausible future reduction is to carry the semantic role inside the -engine's aggregated `RunReport`. That could remove the second role map used to -survive push/pop, but it would also make a benchmark-accounting concept part of -the public cross-crate report type. It is deferred until the role is useful to -the engine itself. Likewise, the extra timers can be gated if this moves -upstream and the measured tax becomes unacceptable; the current control run -does not justify that branch. - -As a review-surface count against `5ead0a0`, the current implementation is net -`+329` Rust source lines including the new 74-line timer module and inline -tests, net `+175` Python report lines, net `+345` external test/snapshot lines, -and net `+53` README lines. These are diff counts rather than runtime -complexity: the Rust report transport itself shrank while replacing V2, and -the snapshots contain no executable logic. The largest irreducible pieces are -the exclusive timing boundaries and their tests; the open-map transport and -mechanism/name projection are the parts deliberately kept small. - -The final driver redesign reduced production Python report complexity -from the pre-amendment net `+218` lines to `+175`: it deleted -`PhaseRollupView`, `_phase_rollups`, the global rollup renderer, endpoint-total -ruleset confidence intervals, the ten-column ruleset table, and the per-table -row-guide styling switch. External -validation grew because it now locks down direct mechanism-parent equality, -per-parent child additivity, deterministic phase threshold, and all report -renderers and supported widths. That growth is test-only; no second runtime -analysis or presentation path remains. - -### Fresh off-versus-term evidence - -The six-round report is: - -```text -/tmp/term-overhead-mechanisms-v3-20260812.jsonl -``` - -All 72 runs succeeded. The displayed suite wall ratio is `2.04–2.09x`. -The report treats endpoint samples as independent because the JSONL has no -persistent round-pair identity. Its additive suite slowdown is: - -| Mechanism | Delta | Share of slowdown | +| Math | 2.01–2.11x | +| eggcc | 1.43–1.48x | +| Pointer analysis | 2.27–2.37x | +| Hardboiled | 1.88–1.94x | +| Luminal | 3.38–3.46x | +| Herbie | 1.96–2.02x | +| Misaal HVX | 2.95–3.07x | +| Churchroad wide multiply | 0.705–0.715x | +| DialEgg NMM40 | 1.64–1.66x | +| SPEQ preserved-reference suite | 3.56–3.61x | + +Churchroad is a useful warning against treating every encoding-induced change +as overhead: its `mapping` Search becomes 309 ms faster, more than offsetting +the added frontend and maintenance work. + +### Timer-tax control + +The extra timers were also measured against a V3-compatible clean control on +the same `5ead0a0` source state. That ten-round, six-workload comparison held +the summary serialization shape fixed while replacing the added timer sites +with zero-valued leaves. The clean suite mean was `1.723830 s`; the instrumented +mean was `1.727558 s`, a `1.00216x` point ratio with a `0.995–1.010x` 95% +interval. This detects no suite-level slowdown and rules out a 5–10% timer tax +for the measured off-mode workload mix. The primary term-versus-off result is +also same-binary, so both of its endpoints pay the retained instrumentation. + +## What “inline rebuilding” can mean + +The report separates two distinct counterfactuals: + +1. **Remove Equality ruleset assembly.** This removes 299 ms of lazy plan + creation and per-invocation executable-ruleset construction, producing an + implied `1.52x` suite ratio. +2. **Make net Equality/rebuild baseline-equivalent.** This removes the entire + 455 ms net responsibility, producing an implied `1.47x` ratio. + +The second number is the optimistic answer to “what if the relational UF and +rebuild were as cheap as native rebuilding?” It is not the cost of one named +ruleset. Encoded maintenance is collective: `@rebuilding`, `@parent`, cleanup, +and subsumption together replace the native rebuild loop. + +Across the suite, generated Equality maintenance adds 863 ms before crediting +the 407 ms of native Rebuild it replaces: + +| Equality phase | Mean delta | +| --- | ---: | +| Assembly | +299 ms | +| Search | +375 ms | +| Apply | +77.8 ms | +| Execution | +13.2 ms | +| Merge | +96.9 ms | +| Native Rebuild replaced | −407 ms | +| **Net Equality/rebuild** | **+455 ms** | + +So a plan-cache or inline-assembly change attacks a real cost, especially on +eggcc, but it leaves most Equality Search/Apply/Merge work intact. Conversely, +folding the native-rebuild credit into `@rebuilding` would falsely make that +single generated ruleset look cheap and obscure the collective substitution. + +## Optimization ceilings + +The report now performs the arithmetic directly. Each row removes only the +named positive candidate-minus-baseline deltas, preserves candidate-side +speedups, and holds every other mean fixed. + +| Hypothetical change | Time removed | Implied ratio | | --- | ---: | ---: | -| Typecheck | +290 ms | 15.7% | -| Frontend/install | +270 ms | 14.6% | -| Program rules | +723 ms | 39.1% | -| Equality/rebuild | +416 ms | 22.5% | -| Commands | +116 ms | 6.28% | -| Residual | +34.0 ms | 1.84% | - -The file rows preserve the earlier diagnosis: Math is 62.5% Equality; -Pointer is 79.1% Typecheck plus Frontend; Luminal is 46.3% Program and only -4.89% Equality; eggcc and Herbie remain mixed. The small residual is the -accounting self-check that the table explains nearly all of the observed -slowdown. - -### Instrumentation-tax control - -The clean-control report is: - -```text -/tmp/term-overhead-timer-tax-leaves-v3-20260812.jsonl -``` - -It compares ten off-mode rounds of this instrumented build against a temporary -clean `5ead0a0` build. The clean build received only a compatibility adapter -that emits the same flat V3 leaf count and serialization shape with the new -process and Assembly values fixed at zero; it does not execute their timers. -This holds timing-summary serialization approximately constant while isolating -the additional timer sites. - -The clean suite mean was `1.723830 s`; the instrumented suite mean was -`1.727558 s`, for an instrumented/clean point ratio of `1.00216x`. The report's -95% interval is `0.995–1.01x` and includes 1. This run therefore detects no -suite-level timer slowdown and rules out a 5–10% tax under the measured -off-mode workload mix. +| Remove added typechecking | 345 ms | 1.51x | +| Remove added frontend/install | 344 ms | 1.51x | +| Remove both frontend groups | 689 ms | 1.40x | +| Remove Equality assembly | 299 ms | 1.52x | +| Remove net Equality/rebuild | 455 ms | 1.47x | +| Remove source-rule execution delta | 563 ms | 1.44x | +| Remove every positive non-program delta | 1.28 s | 1.20x | +| Remove every recorded positive mechanism delta | 1.84 s | 1.02x | + +These are additive accounting ceilings, not implementation predictions. They +have no confidence intervals and do not model interactions: removing generated +types or identities may also change Program Search, Apply, Merge, or plan +assembly. The `1.02x` final row is primarily an accounting-closure check; it +leaves the 46 ms residual rather than pretending uninstrumented time is freely +removable. + +## Workload narratives + +- **Math:** Equality/rebuild is 60.6% of the slowdown. Generated maintenance + costs 395 ms and replaces 135 ms of native rebuild. This is the clearest + relational-UF target, though changed default-rule Apply and Merge still add + 152 ms. +- **eggcc:** no single mechanism wins. Typecheck plus frontend adds 136 ms, + source rules add 108 ms, net Equality adds 83 ms, and commands add 35 ms. + Equality is assembly-heavy: 245 ms of added assembly is almost exactly + offset by 245 ms of removed native rebuild before Search and execution are + counted. `always-run` carries 82 ms of the Program delta. +- **Pointer analysis:** net Equality is largest at 29 ms, frontend is 21 ms, + and Program is 11 ms. Its 9.5 ms residual is large enough that tiny + sub-mechanism conclusions should remain cautious. +- **Hardboiled:** source rules add 33 ms, while typecheck plus frontend adds + 49 ms. The default ruleset's Search dominates its Program child; check + evaluation is routed symmetrically under Commands rather than appearing as a + term-only default ruleset artifact. +- **Luminal:** Program is 403 ms, 46.5% of the slowdown; Equality is only + 39.8 ms, 4.59%. `fusion_grow` and `fusion_pair` add 169 and 147 ms, almost + entirely Search. Typecheck plus frontend adds another 344 ms. UF work is not + the limiting explanation here. +- **Herbie:** mixed across Program (26.6%), Equality (22.4%), frontend/typecheck + (33.1%), and Commands (14.8%). +- **Misaal HVX:** typecheck plus frontend explains 90.6% of the slowdown. + Program and Equality together explain less than 4%; a UF optimization would + barely move it. +- **Churchroad:** Program Search improves by 308 ms net, making term encoding + faster overall despite every other top-level mechanism becoming slower. +- **DialEgg:** Program contributes 57.9%, led by Apply and Merge; net Equality + contributes 16.3%. +- **SPEQ:** Program contributes 74.5%, mostly assembly in four transform + rulesets; Equality is below 1%. + +## What this answers—and what it does not + +The additive report now answers: + +- how much slowdown is frontend, source-rule execution, relational + equality/rebuild, commands, or residual; +- whether Equality cost is assembly or execution; +- which source or maintenance rulesets carry Program and Equality changes; and +- optimistic remaining ratios when selected positive deltas disappear. + +It does not identify why a source rule searches or assembles more slowly. For +Luminal, the data localizes the problem to `fusion_grow`/`fusion_pair` Search, +but distinguishing wider tuples, extra identity columns, changed join order, +or greater state churn requires a profiler or a targeted lowering ablation. +Likewise, the counterfactual rows cannot predict cross-mechanism effects. + +## Measurement design + +Every successful process emits one sorted, open list of exclusive +`path -> nanoseconds` leaves: + +- `typecheck/total`; +- `frontend/{parse,other,install}`; +- `program//`; +- `equality//`; +- `equality/rebuild/` for native rebuild tails; and +- `commands/{actions,check,other}`. + +Rulesets receive an explicit Program or Equality-maintenance role at +declaration time; the report never infers semantics from an `@` prefix. Native +Rebuild and encoded maintenance therefore land under one responsibility before +subtraction. Checks use one command path in both modes. Residual is derived as +wall time minus every recorded leaf and remains the additive self-check. + +The ruleset panel is a literal expansion of the decomposition's two +ruleset-borne columns: + +- `Program rules — own work` excludes source rules' native Rebuild tails; +- `Equality/rebuild — net` contains all maintenance rules and one global + `Native rebuild replaced` child; +- source children are top five by absolute own-work delta plus an exact + `Other`; and +- every nonzero maintenance child is shown. + +Parent rows exactly equal the Program and Equality cells, and children exactly +sum to their parent. No report-time name heuristic or cross-endpoint rebuild +credit is needed. + +## Complexity and minimization + +The retained complexity has five responsibilities: + +| Layer | Required work | Deliberate simplification | +| --- | --- | --- | +| Engine timing | Exclusive process and six-phase ruleset boundaries | Static path slices and one duration map | +| Semantic routing | Program, maintenance, native rebuild, and check ownership | One two-variant role instead of name-prefix inference | +| Transport | Persist exact leaves for later projections | One open segmented-path list; no fixed phase structs or second ruleset schema | +| Analysis | Align endpoint samples, derive residual, mechanisms, drivers, and ceilings | One generic path-sample ledger; parent/child sums are direct | +| Presentation | Decomposition, suite ceilings, and per-file drivers | One shared catalog and table renderer for Rich, Markdown, and interactive output | + +The final reduction pass kept the old execution path recognizable and removed +presentation-only alternatives: there is no global phase-rollup model, no +ten-column ruleset table, no duplicated fixed five-bucket wire record, and no +special report-time native-rebuild credit. The optimization table is derived +from the same means and leaf ledger rather than introducing another recording +shape. Further reduction would either lose the Assembly/Search distinction +that separates plan-cache work from query-shape work or make the signs in the +ruleset panel misleading again. + +## Engineering direction + +The measurements support parallel, falsifiable tracks rather than one “UF +fix”: + +1. eliminate generated parsing, re-typechecking, and installation; +2. cache or fuse Equality assembly, then measure whether Equality execution + can approach native rebuild; +3. restore source-rule physical shapes, especially Luminal Search and SPEQ + assembly; and +4. retain the open ledger while each optimization lands so cross-mechanism + movement remains visible. + +A 5–10% always-on target is possible only if these improvements compose. The +current data rejects both “frontend alone” and “relational UF alone” as +sufficient strategies. From 6d617fec82d459484e6e8173afcf4d4be8f7b9d4 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 00:05:39 -0400 Subject: [PATCH 4/9] Keep timing reports observational --- README.md | 38 +-- benchmarking/reports/analysis.py | 109 +----- benchmarking/reports/presentation.py | 57 +--- term-encoding-overhead-benchmark.md | 317 +++++++++--------- term-encoding-overhead-breakdown.md | 175 +++++----- .../__snapshots__/test_report_rendering.ambr | 43 +-- tests/test_report_analysis.py | 78 ----- tests/test_report_rendering.py | 28 +- 8 files changed, 275 insertions(+), 570 deletions(-) diff --git a/README.md b/README.md index 7b7aa57c..9e890706 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ path. | --- | --- | | `summary` | comparison selection and headline summary | | `files` | per-file wall time and peak RSS estimates | -| `phases` | additive slowdown decomposition plus suite optimization ceilings | +| `phases` | additive suite and per-file slowdown decomposition | | `rulesets` | Program/Equality driver groups and changed rulesets per file | The default is `summary`. For example: @@ -381,25 +381,17 @@ leaf. It includes process setup, reporting, teardown, and any still- uninstrumented work. At `--detail phases`, the additive slowdown-decomposition table has a suite row -and one row per file. Its rendered headers are `Wall Δ`, `Typecheck`, -`Frontend`, `Program`, `Equality`, `Commands`, and `Residual`. Every mechanism -cell displays its share of the wall-time change first, then -candidate-minus-baseline milliseconds. `◆` marks the largest absolute -mechanism share in each row; Rich and interactive reports also bold that cell, -dim contributions below 5%, and color improvements green. Expected overhead is -neutral rather than red; warning and error colors are reserved for suspect -measurements. Percentages may be negative or exceed 100% when mechanisms -offset. `!` on a Residual cell means at least one endpoint's mean recorded total -exceeded its wall time. - -A compact suite-level `Optimization ceilings` table then resets selected -positive candidate-minus-baseline deltas to zero and reports the remaining -wall-time change and implied point ratio. Candidate-side speedups are retained. -It distinguishes removing only Equality ruleset assembly from making the -entire net Equality/rebuild responsibility match the baseline. These rows are -optimistic additive accounting bounds, not predictions: they hold every other -measured mean fixed, omit confidence intervals, and cannot model interactions -between optimizations. Residual is never treated as removable work. +and one row per file. The suite row is the sum of each selected file's +candidate-minus-baseline endpoint mean; it is not a single process observation. +Its rendered headers are `Wall Δ`, `Typecheck`, `Frontend`, `Program`, +`Equality`, `Commands`, and `Residual`. Every mechanism cell displays its share +of the row's wall-time change first, then candidate-minus-baseline milliseconds. +`◆` marks the largest absolute mechanism share in each row; Rich and +interactive reports also bold that cell, dim contributions below 5%, and color +improvements green. Expected overhead is neutral rather than red; warning and +error colors are reserved for suspect measurements. Percentages may be +negative or exceed 100% when mechanisms offset. `!` on a Residual cell means at +least one endpoint's mean recorded total exceeded its wall time. At `--detail rulesets`, one compact driver table appears per file. Its `Program rules — own work` and `Equality/rebuild — net` parent rows exactly @@ -619,9 +611,9 @@ shown. No median or geometric mean is mixed into this minimal headline. A timed-out, failed, or otherwise incomplete selected result invalidates the suite result that depends on it. Valid per-file tail comparisons remain useful -when an unrelated file is incomplete. Mechanism contributions, optimization -ceilings, and ruleset totals or component deltas are descriptive diagnostics; -only endpoint estimates and ratios receive confidence intervals. +when an unrelated file is incomplete. Mechanism contributions and ruleset +totals or component deltas are descriptive diagnostics; only endpoint +estimates and ratios receive confidence intervals. The `<2x` proof goal is established only when the upper bound of the suite wall ratio's 95% confidence interval is below `2x` for a proofs-versus-off diff --git a/benchmarking/reports/analysis.py b/benchmarking/reports/analysis.py index ac428934..cd4811de 100644 --- a/benchmarking/reports/analysis.py +++ b/benchmarking/reports/analysis.py @@ -24,17 +24,6 @@ RulesetPhaseName = Literal["assembly", "search", "apply", "execution", "merge", "rebuild"] RulesetMechanism = Literal["program", "equality"] RulesetRowKind = Literal["aggregate", "ruleset", "native_rebuild", "other"] -OptimizationScenario = Literal[ - "typecheck", - "frontend", - "frontend_and_typecheck", - "equality_assembly", - "equality", - "program", - "non_program", - "all_recorded", -] - type _MetricKey = tuple[int, int, MetricName] type _ObservationKey = tuple[int, int] type _TimingPath = tuple[str, ...] @@ -152,22 +141,12 @@ class RulesetContributorView(NamedTuple): delta: RulesetDelta -class OptimizationCeilingView(NamedTuple): - """One suite-wide accounting counterfactual with no causal-speedup claim.""" - - scenario: OptimizationScenario - reset_delta_ns: float - remaining_delta_ns: float - counterfactual_ratio: float - - class PairReportViewData(NamedTuple): """Typed analysis collections requested by one cumulative detail level.""" summary: tuple[SummaryView, ...] files: tuple[FileComparisonView, ...] decomposition: tuple[SlowdownDecompositionView, ...] - ceilings: tuple[OptimizationCeilingView, ...] rulesets: tuple[RulesetContributorView, ...] @@ -201,17 +180,16 @@ def analyze_pair( summary = _summary_rows(comparison, estimates, file_rows, t_critical) if detail == "summary": - return PairReportViewData(summary, (), (), (), ()) + return PairReportViewData(summary, (), (), ()) if detail == "files": - return PairReportViewData(summary, file_rows, (), (), ()) + return PairReportViewData(summary, file_rows, (), ()) timing = _timing_aggregates(observations) decomposition = _slowdown_decomposition(comparison, timing, issues, estimates) - ceilings = _optimization_ceilings(comparison, timing, estimates, decomposition) if detail == "phases": - return PairReportViewData(summary, file_rows, decomposition, ceilings, ()) + return PairReportViewData(summary, file_rows, decomposition, ()) rulesets = _ruleset_contributors(comparison, timing, issues) - return PairReportViewData(summary, file_rows, decomposition, ceilings, rulesets) + return PairReportViewData(summary, file_rows, decomposition, rulesets) def _selected_observations( @@ -456,85 +434,6 @@ def _slowdown_decomposition( return (suite, *result) -def _optimization_ceilings( - comparison: ComparisonSpec, - timing: dict[_ObservationKey, _TimingAggregate], - metric_estimates: dict[_MetricKey, _MetricEstimate], - decomposition: tuple[SlowdownDecompositionView, ...], -) -> tuple[OptimizationCeilingView, ...]: - """Reset selected suite deltas to zero as optimistic accounting bounds.""" - - scenarios: tuple[OptimizationScenario, ...] = ( - "typecheck", - "frontend", - "frontend_and_typecheck", - "equality_assembly", - "equality", - "program", - "non_program", - "all_recorded", - ) - if decomposition[0].issue is not None: - return () - baseline_points = [ - metric_estimates[(0, file_order, "wall_sec")].estimate.point for file_order in range(len(comparison.files)) - ] - candidate_points = [ - metric_estimates[(1, file_order, "wall_sec")].estimate.point for file_order in range(len(comparison.files)) - ] - if None in baseline_points or None in candidate_points: - return () - - baseline_wall_ns = math.fsum(cast(float, point) for point in baseline_points) * 1_000_000_000.0 - candidate_wall_ns = math.fsum(cast(float, point) for point in candidate_points) * 1_000_000_000.0 - if baseline_wall_ns <= 0 or candidate_wall_ns <= baseline_wall_ns: - return () - - equality_assembly_delta = 0.0 - for file_order in range(len(comparison.files)): - endpoint_means = [] - for endpoint_order in (0, 1): - aggregate = timing[(endpoint_order, file_order)] - paths = [ - path for path in aggregate.paths if len(path) >= 2 and path[0] == "equality" and path[1] == "assembly" - ] - endpoint_means.append(statistics.fmean(_sum_path_samples(aggregate, paths))) - equality_assembly_delta += endpoint_means[1] - endpoint_means[0] - - suite = decomposition[0] - deltas = suite.mechanisms - mechanism_deltas = (deltas.typecheck, deltas.frontend, deltas.program, deltas.equality, deltas.commands) - if any(cell.delta_ns is None for cell in mechanism_deltas): - return () - typecheck = cast(float, deltas.typecheck.delta_ns) - frontend = cast(float, deltas.frontend.delta_ns) - program = cast(float, deltas.program.delta_ns) - equality = cast(float, deltas.equality.delta_ns) - commands = cast(float, deltas.commands.delta_ns) - positive = tuple(max(delta, 0.0) for delta in (typecheck, frontend, program, equality, commands)) - typecheck_added, frontend_added, program_added, equality_added, commands_added = positive - reset_deltas = ( - typecheck_added, - frontend_added, - typecheck_added + frontend_added, - max(equality_assembly_delta, 0.0), - equality_added, - program_added, - typecheck_added + frontend_added + equality_added + commands_added, - math.fsum(positive), - ) - wall_delta = candidate_wall_ns - baseline_wall_ns - return tuple( - OptimizationCeilingView( - scenario, - reset_delta, - wall_delta - reset_delta, - (candidate_wall_ns - reset_delta) / baseline_wall_ns, - ) - for scenario, reset_delta in zip(scenarios, reset_deltas, strict=True) - ) - - def _timing_aggregates( observations: dict[_ObservationKey, tuple[IndexedRecord, ...]], ) -> dict[_ObservationKey, _TimingAggregate]: diff --git a/benchmarking/reports/presentation.py b/benchmarking/reports/presentation.py index cae73e58..8a6360f0 100644 --- a/benchmarking/reports/presentation.py +++ b/benchmarking/reports/presentation.py @@ -19,7 +19,6 @@ Estimate, FileComparisonView, MetricName, - OptimizationCeilingView, PairReportViewData, RatioEstimate, ResultClass, @@ -56,7 +55,8 @@ } RATIO_DIRECTION = "Ratios are candidate / baseline; below 1 is lower and above 1 is higher." DECOMPOSITION_CAPTION = ( - "Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. " + "The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. " + "Each mechanism cell is its share of that row's wall-time change followed by its signed mean time change. " "Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every " "phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with " "native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or " @@ -74,13 +74,6 @@ "Important phases include every |phase Δ| ≥ max(1 ms, 10% of |row Δ|), always include the dominant phase (◆), " "and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases." ) -OPTIMIZATION_CAPTION = ( - "Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other " - "measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while " - "net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting " - "bounds, not implementation predictions; point ratios have no confidence intervals and interactions can " - "invalidate them. Residual is never removed." -) def build_report_catalog( @@ -100,7 +93,7 @@ def build_report_catalog( if _includes(detail, "files"): sections.append(_files_section(views.files, comparison, file_labels)) if _includes(detail, "phases"): - sections.append(_phases_section(views.decomposition, views.ceilings, comparison, file_labels)) + sections.append(_phases_section(views.decomposition, comparison, file_labels)) if _includes(detail, "rulesets"): sections.append(_rulesets_section(views, comparison, file_labels)) return ReportCatalog(tuple(sections)) @@ -327,7 +320,6 @@ def _files_section( def _phases_section( rows: Sequence[SlowdownDecompositionView], - ceilings: Sequence[OptimizationCeilingView], comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: @@ -335,7 +327,8 @@ def _phases_section( for row in rows: if row.file_order is None: row_id = report_id("row", "phases", "suite") - label = "Suite total" + file_count = len(comparison.files) + label = f"Suite total ({file_count} {'file' if file_count == 1 else 'files'})" else: file = comparison.files[row.file_order] row_id = report_id("row", "phases", file.sha256, file.fact_directory_sha256) @@ -382,45 +375,7 @@ def _phases_section( caption=DECOMPOSITION_CAPTION, alignments=("left", "right", "right", "right", "right", "right", "right", "right"), ) - scenario_labels = { - "typecheck": "Remove added typechecking time", - "frontend": "Remove added frontend/install time", - "frontend_and_typecheck": "Remove added typechecking + frontend time", - "equality_assembly": "Remove added Equality assembly time", - "equality": "Remove added net Equality/rebuild time", - "program": "Remove added source-rule execution time", - "non_program": "Remove every added non-program mechanism", - "all_recorded": "Remove every recorded added mechanism", - } - ceiling_rows = tuple( - _row( - report_id("row", "phases", "ceiling", row.scenario), - text_cell(row.scenario, scenario_labels[row.scenario]), - text_cell(row.reset_delta_ns, format_duration(row.reset_delta_ns, signed=True)), - text_cell( - row.remaining_delta_ns, - _format_delta_ms(row.remaining_delta_ns), - tone=_delta_tone(row.remaining_delta_ns), - ), - text_cell( - row.counterfactual_ratio, - f"{_three_significant_digits(row.counterfactual_ratio)}x", - tone="positive" if row.counterfactual_ratio < 1 else "default", - ), - ) - for row in ceilings - ) - ceiling_table = _table( - report_id("table", "phases", "optimization-ceilings"), - "Optimization ceilings", - ("scenario", "reset_delta", "remaining_delta", "counterfactual_ratio"), - ("Hypothetical change", "Time removed", "Remaining wall Δ", "Implied ratio"), - ceiling_rows, - caption=OPTIMIZATION_CAPTION, - alignments=("left", "right", "right", "right"), - ) - blocks = (table, ceiling_table) if ceilings else (table,) - return ReportSection("phases", "Slowdown decomposition", blocks) + return ReportSection("phases", "Slowdown decomposition", (table,)) def _slowdown_cell(cell: SlowdownCell, *, leader: bool, warning: bool) -> ReportCell: diff --git a/term-encoding-overhead-benchmark.md b/term-encoding-overhead-benchmark.md index e1984b5b..3c258d20 100644 --- a/term-encoding-overhead-benchmark.md +++ b/term-encoding-overhead-benchmark.md @@ -4,20 +4,20 @@ | Role | Target | Git | Treatment | | --- | --- | --- | --- | -| Baseline | d60202f64424 | d60202f64424 | off | -| Candidate | d60202f64424 | d60202f64424 | term | +| Baseline | d60202f644249ef565de0ca7c51871fe497440a2 | d60202f64424 | off | +| Candidate | d60202f644249ef565de0ca7c51871fe497440a2 | d60202f64424 | term | *10 file(s): math-microbenchmark-rational.egg, eggcc-2mm-pass1.egg, pointer-analysis-initdb.egg (facts: /Users/saul/p/wt/egglog-encoding/term-encoding-always-on/egglog/tests/pointer-analysis-initdb), hardboiled_conv1d_32.egg, luminal-llama.egg, herbie.egg, misaal-hvx-dot-product.egg, churchroad-wide-multiply.egg, dialegg-nmm40.egg, speq-preserved-reference-suite.egg · 6 round(s) per endpoint/file · 120 s timeout per run · Report: /private/tmp/term-encoding-d60202f.jsonl* -## Summary — d60202f64424 term vs d60202f64424 off +## Summary — d60202f644249ef565de0ca7c51871fe497440a2 term vs d60202f644249ef565de0ca7c51871fe497440a2 off | Metric | Scope | File(s) | Ratio (95% CI) | Result | | --- | --- | --- | ---: | --- | -| Wall time | Suite total | 10 files | 1.61–1.63x | slower | -| Wall time | Lowest-ratio file | churchroad-wide-multiply.egg | 0.705–0.715x | faster | -| Wall time | Highest-ratio file | speq-preserved-reference-suite.egg | 3.56–3.61x | slower | -| Peak RSS | Lowest-ratio file | churchroad-wide-multiply.egg | 1.10–1.11x | higher RSS | -| Peak RSS | Highest-ratio file | pointer-analysis-initdb.egg | 3.59–3.65x | higher RSS | +| Wall time | Suite total | 10 files | 1.62–1.64x | slower | +| Wall time | Lowest-ratio file | churchroad-wide-multiply.egg | 0.708–0.720x | faster | +| Wall time | Highest-ratio file | speq-preserved-reference-suite.egg | 3.55–3.65x | slower | +| Peak RSS | Lowest-ratio file | churchroad-wide-multiply.egg | 1.11–1.13x | higher RSS | +| Peak RSS | Highest-ratio file | pointer-analysis-initdb.egg | 3.78–3.80x | higher RSS | *Ratios are candidate / baseline; below 1 is lower and above 1 is higher.* @@ -27,64 +27,49 @@ | File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | | --- | ---: | ---: | ---: | --- | -| math-microbenchmark-rational.egg | 393–412 ms | 825–836 ms | 2.01–2.11x | slower | -| eggcc-2mm-pass1.egg | 799–812 ms | 1.16–1.19 s | 1.43–1.48x | slower | -| pointer-analysis-initdb.egg | 57.1–58.4 ms | 132–137 ms | 2.27–2.37x | slower | -| hardboiled_conv1d_32.egg | 110–112 ms | 210–215 ms | 1.88–1.94x | slower | -| luminal-llama.egg | 355–363 ms | 1.23–1.23 s | 3.38–3.46x | slower | -| herbie.egg | 51.7–52.4 ms | 102–105 ms | 1.96–2.02x | slower | -| misaal-hvx-dot-product.egg | 33.2–33.9 ms | 99.3–103 ms | 2.95–3.07x | slower | -| churchroad-wide-multiply.egg | 1.00–1.02 s | 715–717 ms | 0.705–0.715x | faster | -| dialegg-nmm40.egg | 158–160 ms | 261–263 ms | 1.64–1.66x | slower | -| speq-preserved-reference-suite.egg | 45.7–46.0 ms | 163–165 ms | 3.56–3.61x | slower | +| math-microbenchmark-rational.egg | 413–431 ms | 844–865 ms | 1.98–2.08x | slower | +| eggcc-2mm-pass1.egg | 819–824 ms | 1.19–1.21 s | 1.45–1.48x | slower | +| pointer-analysis-initdb.egg | 58.0–59.8 ms | 135–137 ms | 2.27–2.35x | slower | +| hardboiled_conv1d_32.egg | 112–113 ms | 215–218 ms | 1.91–1.95x | slower | +| luminal-llama.egg | 363–366 ms | 1.24–1.25 s | 3.39–3.43x | slower | +| herbie.egg | 52.7–54.3 ms | 105–108 ms | 1.95–2.03x | slower | +| misaal-hvx-dot-product.egg | 33.9–34.5 ms | 102–104 ms | 2.97–3.05x | slower | +| churchroad-wide-multiply.egg | 1.00–1.01 s | 714–724 ms | 0.708–0.720x | faster | +| dialegg-nmm40.egg | 158–165 ms | 263–270 ms | 1.61–1.69x | slower | +| speq-preserved-reference-suite.egg | 46.6–47.6 ms | 168–171 ms | 3.55–3.65x | slower | ### Peak RSS | File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | | --- | ---: | ---: | ---: | --- | -| math-microbenchmark-rational.egg | 287.1–287.3 MiB | 494.0–494.6 MiB | 1.72–1.72x | higher RSS | -| eggcc-2mm-pass1.egg | 109.4–110.8 MiB | 248.2–250.2 MiB | 2.25–2.28x | higher RSS | -| pointer-analysis-initdb.egg | 41.7–42.4 MiB | 152.1–152.7 MiB | 3.59–3.65x | higher RSS | -| hardboiled_conv1d_32.egg | 41.6–41.9 MiB | 68.1–68.6 MiB | 1.63–1.64x | higher RSS | -| luminal-llama.egg | 118.1–119.4 MiB | 256.4–259.8 MiB | 2.15–2.19x | higher RSS | -| herbie.egg | 19.9–20.1 MiB | 34.0–34.2 MiB | 1.69–1.71x | higher RSS | -| misaal-hvx-dot-product.egg | 31.7–31.9 MiB | 65.5–65.9 MiB | 2.06–2.07x | higher RSS | -| churchroad-wide-multiply.egg | 20.5–20.6 MiB | 22.6–22.8 MiB | 1.10–1.11x | higher RSS | -| dialegg-nmm40.egg | 30.6–30.8 MiB | 97.8–98.0 MiB | 3.17–3.20x | higher RSS | -| speq-preserved-reference-suite.egg | 16.8–17.1 MiB | 35.2–35.7 MiB | 2.07–2.11x | higher RSS | +| math-microbenchmark-rational.egg | 287.4–287.6 MiB | 470.3–470.5 MiB | 1.64–1.64x | higher RSS | +| eggcc-2mm-pass1.egg | 106.6–110.5 MiB | 242.9–248.6 MiB | 2.22–2.31x | higher RSS | +| pointer-analysis-initdb.egg | 40.2–40.3 MiB | 152.2–152.7 MiB | 3.78–3.80x | higher RSS | +| hardboiled_conv1d_32.egg | 41.4–41.8 MiB | 68.0–68.3 MiB | 1.63–1.65x | higher RSS | +| luminal-llama.egg | 117.1–119.9 MiB | 258.4–261.0 MiB | 2.16–2.22x | higher RSS | +| herbie.egg | 20.0–20.1 MiB | 33.7–33.9 MiB | 1.68–1.69x | higher RSS | +| misaal-hvx-dot-product.egg | 31.5–32.1 MiB | 64.9–66.2 MiB | 2.03–2.09x | higher RSS | +| churchroad-wide-multiply.egg | 20.1–20.5 MiB | 22.6–22.8 MiB | 1.11–1.13x | higher RSS | +| dialegg-nmm40.egg | 30.7–31.0 MiB | 98.1–98.3 MiB | 3.17–3.20x | higher RSS | +| speq-preserved-reference-suite.egg | 16.7–17.1 MiB | 35.2–35.4 MiB | 2.07–2.11x | higher RSS | ## Slowdown decomposition | File | Wall Δ | Typecheck | Frontend | Program | Equality | Commands | Residual | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Suite total | +1889 ms | +18.2% +345 ms | +18.2% +344 ms | ◆ +29.8% +563 ms | +24.1% +455 ms | +7.22% +136 ms | +2.46% +46.4 ms | -| math-microbenchmark-rational.egg | +428 ms | +0.369% +1.58 ms | +0.387% +1.65 ms | +35.5% +152 ms | ◆ +60.6% +259 ms | +2.76% +11.8 ms | +0.397% +1.70 ms | -| eggcc-2mm-pass1.egg | +368 ms | +18.0% +66.2 ms | +18.9% +69.4 ms | ◆ +29.4% +108 ms | +22.7% +83.4 ms | +9.37% +34.5 ms | +1.68% +6.18 ms | -| pointer-analysis-initdb.egg | +76.4 ms | +2.88% +2.20 ms | +27.0% +20.7 ms | +14.9% +11.4 ms | ◆ +38.4% +29.3 ms | +4.36% +3.34 ms | +12.5% +9.53 ms | -| hardboiled_conv1d_32.egg | +101 ms | +24.6% +24.9 ms | +23.4% +23.7 ms | ◆ +32.9% +33.3 ms | +10.6% +10.7 ms | +5.65% +5.72 ms | +2.83% +2.86 ms | -| luminal-llama.egg | +868 ms | +21.0% +183 ms | +18.6% +162 ms | ◆ +46.5% +403 ms | +4.59% +39.8 ms | +7.43% +64.5 ms | +1.83% +15.9 ms | -| herbie.egg | +51.6 ms | +15.4% +7.96 ms | +17.7% +9.12 ms | ◆ +26.6% +13.7 ms | +22.4% +11.5 ms | +14.8% +7.62 ms | +3.23% +1.67 ms | -| misaal-hvx-dot-product.egg | +67.4 ms | +45.1% +30.4 ms | ◆ +45.5% +30.7 ms | +1.07% +0.724 ms | +2.81% +1.90 ms | +0.702% +0.473 ms | +4.86% +3.28 ms | -| churchroad-wide-multiply.egg | -292 ms | -1.90% +5.56 ms | -2.11% +6.18 ms | ◆ +105% -308 ms | -0.552% +1.61 ms | -0.150% +0.439 ms | -0.496% +1.45 ms | -| dialegg-nmm40.egg | +103 ms | +11.1% +11.4 ms | +9.85% +10.1 ms | ◆ +57.9% +59.5 ms | +16.3% +16.8 ms | +2.71% +2.78 ms | +2.11% +2.17 ms | -| speq-preserved-reference-suite.egg | +118 ms | +9.95% +11.8 ms | +9.13% +10.8 ms | ◆ +74.5% +88.2 ms | +0.660% +0.782 ms | +4.37% +5.18 ms | +1.41% +1.67 ms | - -*Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* - -### Optimization ceilings - -| Hypothetical change | Time removed | Remaining wall Δ | Implied ratio | -| --- | ---: | ---: | ---: | -| Remove added typechecking time | +345 ms | +1545 ms | 1.51x | -| Remove added frontend/install time | +344 ms | +1545 ms | 1.51x | -| Remove added typechecking + frontend time | +689 ms | +1201 ms | 1.40x | -| Remove added Equality assembly time | +299 ms | +1590 ms | 1.52x | -| Remove added net Equality/rebuild time | +455 ms | +1434 ms | 1.47x | -| Remove added source-rule execution time | +563 ms | +1327 ms | 1.44x | -| Remove every added non-program mechanism | +1.28 s | +609 ms | 1.20x | -| Remove every recorded added mechanism | +1.84 s | +46.4 ms | 1.02x | - -*Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is never removed.* +| Suite total (10 files) | +1935 ms | +18.1% +350 ms | +17.9% +346 ms | ◆ +29.7% +575 ms | +24.6% +476 ms | +7.12% +138 ms | +2.60% +50.3 ms | +| math-microbenchmark-rational.egg | +433 ms | +0.385% +1.66 ms | +0.417% +1.80 ms | +33.6% +145 ms | ◆ +62.5% +270 ms | +2.64% +11.4 ms | +0.464% +2.01 ms | +| eggcc-2mm-pass1.egg | +380 ms | +17.9% +68.2 ms | +18.6% +70.7 ms | ◆ +29.8% +113 ms | +22.7% +86.3 ms | +9.15% +34.8 ms | +1.80% +6.84 ms | +| pointer-analysis-initdb.egg | +77.3 ms | +2.90% +2.25 ms | +26.9% +20.8 ms | +14.8% +11.4 ms | ◆ +38.7% +29.9 ms | +3.66% +2.83 ms | +13.0% +10.1 ms | +| hardboiled_conv1d_32.egg | +104 ms | +24.2% +25.2 ms | +23.1% +24.1 ms | ◆ +33.2% +34.7 ms | +10.8% +11.3 ms | +5.77% +6.02 ms | +2.83% +2.96 ms | +| luminal-llama.egg | +879 ms | +21.0% +185 ms | +18.1% +159 ms | ◆ +46.4% +408 ms | +5.10% +44.8 ms | +7.40% +65.1 ms | +2.01% +17.7 ms | +| herbie.egg | +53.0 ms | +15.3% +8.11 ms | +17.5% +9.29 ms | ◆ +26.6% +14.1 ms | +22.3% +11.8 ms | +15.4% +8.16 ms | +2.86% +1.52 ms | +| misaal-hvx-dot-product.egg | +68.6 ms | +44.6% +30.6 ms | ◆ +45.7% +31.4 ms | +1.16% +0.794 ms | +2.90% +1.99 ms | +0.709% +0.487 ms | +4.93% +3.39 ms | +| churchroad-wide-multiply.egg | -288 ms | -1.97% +5.67 ms | -2.21% +6.36 ms | ◆ +105% -303 ms | -0.606% +1.74 ms | -0.163% +0.469 ms | -0.496% +1.43 ms | +| dialegg-nmm40.egg | +105 ms | +11.0% +11.5 ms | +9.93% +10.4 ms | ◆ +57.8% +60.5 ms | +16.3% +17.1 ms | +2.75% +2.88 ms | +2.20% +2.30 ms | +| speq-preserved-reference-suite.egg | +122 ms | +9.89% +12.1 ms | +9.27% +11.3 ms | ◆ +73.9% +90.4 ms | +0.691% +0.846 ms | +4.52% +5.52 ms | +1.76% +2.15 ms | + +*The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. Each mechanism cell is its share of that row's wall-time change followed by its signed mean time change. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* ## Ruleset drivers @@ -94,14 +79,14 @@ | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +152 ms | +35.5% | ◆ Apply +80.8 ms; Merge +69.9 ms; … | -| ↳ | +152 ms | | ◆ Apply +80.8 ms; Merge +69.9 ms; … | -| Equality/rebuild — net | +259 ms | +60.6% | ◆ Search +267 ms; Apply +57.6 ms; Merge +69.4 ms; Rebuild -135 ms; … | -| ↳ @rebuilding | +354 ms | | ◆ Search +233 ms; Apply +57.2 ms; Merge +62.8 ms; … | -| ↳ @parent | +41.0 ms | | ◆ Search +33.9 ms; Merge +6.60 ms; … | -| ↳ @rebuilding_cleanup | +855 ns | | ◆ Assembly +855 ns | -| ↳ @subsume_ruleset | +209 ns | | ◆ Assembly +209 ns | -| ↳ Native rebuild replaced | -135 ms | | ◆ Rebuild -135 ms | +| Program rules — own work | +145 ms | +33.6% | ◆ Apply +73.1 ms; Merge +71.1 ms; … | +| ↳ | +145 ms | | ◆ Apply +73.1 ms; Merge +71.1 ms; … | +| Equality/rebuild — net | +270 ms | +62.5% | ◆ Search +280 ms; Apply +59.9 ms; Merge +72.1 ms; Rebuild -142 ms; … | +| ↳ @rebuilding | +371 ms | | ◆ Search +246 ms; Apply +59.4 ms; Merge +65.3 ms; … | +| ↳ @parent | +41.2 ms | | ◆ Search +33.8 ms; Merge +6.83 ms; … | +| ↳ @rebuilding_cleanup | +908 ns | | ◆ Assembly +908 ns | +| ↳ @subsume_ruleset | +223 ns | | ◆ Assembly +223 ns | +| ↳ Native rebuild replaced | -142 ms | | ◆ Rebuild -142 ms | *Program + Equality account for +96.1% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* @@ -109,86 +94,86 @@ | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +108 ms | +29.4% | Assembly +11.1 ms; ◆ Search +54.5 ms; Apply +24.5 ms; Merge +17.7 ms; … | -| ↳ always-run | +82.2 ms | | Assembly +8.52 ms; ◆ Search +43.2 ms; Apply +18.1 ms; Merge +12.0 ms; … | -| ↳ type-analysis | +7.08 ms | | Search +1.53 ms; ◆ Apply +2.84 ms; Merge +2.39 ms; … | -| ↳ is-resolved | +4.99 ms | | ◆ Search +4.01 ms; … | -| ↳ terms | +3.52 ms | | ◆ Search +1.46 ms; … | -| ↳ terms-helpers | +3.48 ms | | ◆ Search +1.86 ms; … | -| ↳ Other (23 more source rulesets) | +7.06 ms | | Assembly +1.47 ms; ◆ Search +2.44 ms; Apply +1.35 ms; Merge +1.68 ms; … | -| Equality/rebuild — net | +83.4 ms | +22.7% | ◆ Assembly +245 ms; Search +55.5 ms; Execution +10.8 ms; Merge +10.6 ms; Rebuild -245 ms; … | -| ↳ @rebuilding | +277 ms | | ◆ Assembly +201 ms; Search +50.1 ms; … | -| ↳ @parent | +48.6 ms | | ◆ Assembly +41.6 ms; Search +5.31 ms; … | -| ↳ @subsume_ruleset | +2.78 ms | | ◆ Assembly +2.68 ms; … | -| ↳ @rebuilding_cleanup | +65.8 us | | ◆ Assembly +65.8 us | -| ↳ Native rebuild replaced | -245 ms | | ◆ Rebuild -245 ms | - -*Program + Equality account for +52.1% of this file's wall-time change. Source rules shown: 5/28 plus exact Other. Maintenance rules shown: 4/4.* +| Program rules — own work | +113 ms | +29.8% | Assembly +11.6 ms; ◆ Search +56.8 ms; Apply +25.5 ms; Merge +18.8 ms; … | +| ↳ always-run | +85.6 ms | | Assembly +8.73 ms; ◆ Search +44.5 ms; Apply +19.1 ms; Merge +12.9 ms; … | +| ↳ type-analysis | +8.36 ms | | Search +2.43 ms; ◆ Apply +2.93 ms; Merge +2.47 ms; … | +| ↳ is-resolved | +5.00 ms | | ◆ Search +4.02 ms; … | +| ↳ terms | +3.65 ms | | ◆ Search +1.46 ms; … | +| ↳ terms-helpers | +3.61 ms | | ◆ Search +1.90 ms; … | +| ↳ Other (23 more source rulesets) | +7.11 ms | | Assembly +1.49 ms; ◆ Search +2.46 ms; Apply +1.34 ms; Merge +1.73 ms; … | +| Equality/rebuild — net | +86.3 ms | +22.7% | ◆ Assembly +248 ms; Search +57.6 ms; Execution +11.4 ms; Merge +11.2 ms; Rebuild -247 ms; … | +| ↳ @rebuilding | +282 ms | | ◆ Assembly +203 ms; Search +52.0 ms; … | +| ↳ @parent | +49.3 ms | | ◆ Assembly +42.0 ms; Search +5.56 ms; … | +| ↳ @subsume_ruleset | +2.83 ms | | ◆ Assembly +2.73 ms; … | +| ↳ @rebuilding_cleanup | +67.2 us | | ◆ Assembly +67.2 us | +| ↳ Native rebuild replaced | -247 ms | | ◆ Rebuild -247 ms | + +*Program + Equality account for +52.5% of this file's wall-time change. Source rules shown: 5/28 plus exact Other. Maintenance rules shown: 4/4.* ### Ruleset drivers — pointer-analysis-initdb.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +11.4 ms | +14.9% | Apply +3.88 ms; ◆ Merge +7.28 ms; … | -| ↳ | +11.4 ms | | Apply +3.88 ms; ◆ Merge +7.28 ms; … | -| Equality/rebuild — net | +29.3 ms | +38.4% | ◆ Search +19.9 ms; Apply +3.84 ms; Merge +9.41 ms; Rebuild -4.56 ms; … | -| ↳ @rebuilding | +18.9 ms | | ◆ Search +12.7 ms; Apply +3.25 ms; Merge +2.63 ms; … | -| ↳ @parent | +15.0 ms | | ◆ Search +7.19 ms; Merge +6.78 ms; … | -| ↳ @rebuilding_cleanup | +1.29 us | | ◆ Assembly +1.29 us | -| ↳ @subsume_ruleset | +770 ns | | ◆ Assembly +770 ns | -| ↳ Native rebuild replaced | -4.56 ms | | ◆ Rebuild -4.56 ms | +| Program rules — own work | +11.4 ms | +14.8% | Apply +3.91 ms; ◆ Merge +7.45 ms; … | +| ↳ | +11.4 ms | | Apply +3.91 ms; ◆ Merge +7.45 ms; … | +| Equality/rebuild — net | +29.9 ms | +38.7% | ◆ Search +20.2 ms; Apply +3.89 ms; Merge +9.65 ms; Rebuild -4.66 ms; … | +| ↳ @rebuilding | +19.2 ms | | ◆ Search +12.8 ms; Apply +3.29 ms; Merge +2.69 ms; … | +| ↳ @parent | +15.4 ms | | ◆ Search +7.39 ms; Merge +6.96 ms; … | +| ↳ @rebuilding_cleanup | +1.19 us | | ◆ Assembly +1.19 us | +| ↳ @subsume_ruleset | +785 ns | | ◆ Assembly +785 ns | +| ↳ Native rebuild replaced | -4.66 ms | | ◆ Rebuild -4.66 ms | -*Program + Equality account for +53.3% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* +*Program + Equality account for +53.4% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* ### Ruleset drivers — hardboiled_conv1d_32.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +33.3 ms | +32.9% | ◆ Search +23.6 ms; Apply +4.44 ms; … | -| ↳ | +31.8 ms | | ◆ Search +23.6 ms; Apply +3.85 ms; … | -| ↳ typechecking | +967 us | | ◆ Apply +590 us; … | -| ↳ amx | +497 us | | ◆ Assembly +497 us | -| Equality/rebuild — net | +10.7 ms | +10.6% | Assembly +4.74 ms; ◆ Search +5.76 ms; Apply +2.32 ms; Rebuild -3.73 ms; … | -| ↳ @rebuilding | +12.4 ms | | Assembly +3.87 ms; ◆ Search +4.82 ms; Apply +2.29 ms; … | -| ↳ @parent | +2.02 ms | | ◆ Search +943 us; … | -| ↳ @subsume_ruleset | +21.1 us | | ◆ Assembly +21.1 us | -| ↳ @rebuilding_cleanup | +2.68 us | | ◆ Assembly +2.68 us | -| ↳ Native rebuild replaced | -3.73 ms | | ◆ Rebuild -3.73 ms | - -*Program + Equality account for +43.5% of this file's wall-time change. Source rules shown: 3/3. Maintenance rules shown: 4/4.* +| Program rules — own work | +34.7 ms | +33.2% | ◆ Search +24.2 ms; Apply +4.64 ms; … | +| ↳ | +33.1 ms | | ◆ Search +24.2 ms; Apply +4.02 ms; … | +| ↳ typechecking | +1.06 ms | | ◆ Apply +616 us; … | +| ↳ amx | +531 us | | ◆ Assembly +531 us | +| Equality/rebuild — net | +11.3 ms | +10.8% | Assembly +4.99 ms; ◆ Search +5.96 ms; Apply +2.38 ms; Rebuild -3.75 ms; … | +| ↳ @rebuilding | +12.9 ms | | Assembly +4.07 ms; ◆ Search +4.97 ms; Apply +2.34 ms; … | +| ↳ @parent | +2.12 ms | | ◆ Search +984 us; … | +| ↳ @subsume_ruleset | +22.1 us | | ◆ Assembly +22.1 us | +| ↳ @rebuilding_cleanup | +2.88 us | | ◆ Assembly +2.88 us | +| ↳ Native rebuild replaced | -3.75 ms | | ◆ Rebuild -3.75 ms | + +*Program + Equality account for +44.1% of this file's wall-time change. Source rules shown: 3/3. Maintenance rules shown: 4/4.* ### Ruleset drivers — luminal-llama.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +403 ms | +46.5% | Assembly +87.7 ms; ◆ Search +311 ms; … | -| ↳ fusion_grow | +169 ms | | ◆ Search +168 ms; … | -| ↳ fusion_pair | +147 ms | | ◆ Search +145 ms; … | -| ↳ direct_kernel | +36.3 ms | | ◆ Search +36.1 ms; … | -| ↳ matmul_backend | +29.9 ms | | ◆ Assembly +75.1 ms; Search -45.5 ms; … | -| ↳ fusion_merge | +14.7 ms | | ◆ Search +14.2 ms; … | -| ↳ Other (11 more source rulesets) | +7.04 ms | | ◆ Assembly +11.4 ms; Search -5.95 ms; … | -| Equality/rebuild — net | +39.8 ms | +4.59% | ◆ Assembly +42.3 ms; Rebuild -5.66 ms; … | -| ↳ @rebuilding | +44.7 ms | | ◆ Assembly +41.6 ms; … | -| ↳ @parent | +565 us | | ◆ Assembly +514 us; … | -| ↳ @subsume_ruleset | +217 us | | ◆ Assembly +133 us; … | -| ↳ @rebuilding_cleanup | +3.66 us | | ◆ Assembly +3.66 us | -| ↳ Native rebuild replaced | -5.66 ms | | ◆ Rebuild -5.66 ms | - -*Program + Equality account for +51.1% of this file's wall-time change. Source rules shown: 5/16 plus exact Other. Maintenance rules shown: 4/4.* +| Program rules — own work | +408 ms | +46.4% | Assembly +89.0 ms; ◆ Search +314 ms; … | +| ↳ fusion_grow | +170 ms | | ◆ Search +168 ms; … | +| ↳ fusion_pair | +148 ms | | ◆ Search +146 ms; … | +| ↳ direct_kernel | +36.9 ms | | ◆ Search +36.7 ms; … | +| ↳ matmul_backend | +30.5 ms | | ◆ Assembly +76.1 ms; Search -45.9 ms; … | +| ↳ fusion_merge | +14.9 ms | | ◆ Search +14.3 ms; … | +| ↳ Other (11 more source rulesets) | +7.62 ms | | ◆ Assembly +11.8 ms; Search -5.93 ms; … | +| Equality/rebuild — net | +44.8 ms | +5.10% | ◆ Assembly +47.3 ms; Rebuild -5.74 ms; … | +| ↳ @rebuilding | +49.7 ms | | ◆ Assembly +46.5 ms; … | +| ↳ @parent | +627 us | | ◆ Assembly +574 us; … | +| ↳ @subsume_ruleset | +238 us | | ◆ Assembly +147 us; … | +| ↳ @rebuilding_cleanup | +3.58 us | | ◆ Assembly +3.58 us | +| ↳ Native rebuild replaced | -5.74 ms | | ◆ Rebuild -5.74 ms | + +*Program + Equality account for +51.5% of this file's wall-time change. Source rules shown: 5/16 plus exact Other. Maintenance rules shown: 4/4.* ### Ruleset drivers — herbie.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +13.7 ms | +26.6% | ◆ Assembly +8.06 ms; Apply +3.05 ms; Merge +1.98 ms; … | -| ↳ | +13.7 ms | | ◆ Assembly +8.06 ms; Apply +3.05 ms; Merge +1.98 ms; … | -| Equality/rebuild — net | +11.5 ms | +22.4% | ◆ Search +8.05 ms; Apply +1.88 ms; Merge +2.25 ms; Rebuild -1.98 ms; … | -| ↳ @rebuilding | +11.3 ms | | ◆ Search +6.58 ms; Apply +1.80 ms; Merge +1.76 ms; … | -| ↳ @parent | +2.20 ms | | ◆ Search +1.47 ms; … | -| ↳ @rebuilding_cleanup | +2.40 us | | ◆ Assembly +2.40 us | -| ↳ @subsume_ruleset | +1.55 us | | ◆ Assembly +1.55 us | -| ↳ Native rebuild replaced | -1.98 ms | | ◆ Rebuild -1.98 ms | +| Program rules — own work | +14.1 ms | +26.6% | ◆ Assembly +8.33 ms; Apply +3.13 ms; Merge +2.02 ms; … | +| ↳ | +14.1 ms | | ◆ Assembly +8.33 ms; Apply +3.13 ms; Merge +2.02 ms; … | +| Equality/rebuild — net | +11.8 ms | +22.3% | ◆ Search +8.17 ms; Apply +1.92 ms; Merge +2.32 ms; Rebuild -2.01 ms; … | +| ↳ @rebuilding | +11.6 ms | | ◆ Search +6.68 ms; Apply +1.83 ms; Merge +1.82 ms; … | +| ↳ @parent | +2.24 ms | | ◆ Search +1.49 ms; … | +| ↳ @rebuilding_cleanup | +2.48 us | | ◆ Assembly +2.48 us | +| ↳ @subsume_ruleset | +1.60 us | | ◆ Assembly +1.60 us | +| ↳ Native rebuild replaced | -2.01 ms | | ◆ Rebuild -2.01 ms | *Program + Equality account for +48.9% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* @@ -196,31 +181,31 @@ | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +724 us | +1.07% | ◆ Assembly +623 us; … | -| ↳ | +724 us | | ◆ Assembly +623 us; … | -| Equality/rebuild — net | +1.90 ms | +2.81% | ◆ Assembly +1.49 ms; … | -| ↳ @rebuilding | +2.03 ms | | ◆ Assembly +1.48 ms; … | -| ↳ @parent | +48.1 us | | ◆ Search +24.3 us; … | -| ↳ @rebuilding_cleanup | +230 ns | | ◆ Assembly +230 ns | -| ↳ @subsume_ruleset | +90.2 ns | | ◆ Assembly +90.2 ns | -| ↳ Native rebuild replaced | -180 us | | ◆ Rebuild -180 us | +| Program rules — own work | +794 us | +1.16% | ◆ Assembly +675 us; … | +| ↳ | +794 us | | ◆ Assembly +675 us; … | +| Equality/rebuild — net | +1.99 ms | +2.90% | ◆ Assembly +1.57 ms; … | +| ↳ @rebuilding | +2.13 ms | | ◆ Assembly +1.56 ms; … | +| ↳ @parent | +48.8 us | | ◆ Search +25.0 us; … | +| ↳ @rebuilding_cleanup | +223 ns | | ◆ Assembly +223 ns | +| ↳ @subsume_ruleset | +180 ns | | ◆ Assembly +180 ns | +| ↳ Native rebuild replaced | -185 us | | ◆ Rebuild -185 us | -*Program + Equality account for +3.89% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* +*Program + Equality account for +4.06% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* ### Ruleset drivers — churchroad-wide-multiply.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | -308 ms | +105% | ◆ Search -309 ms; … | -| ↳ mapping | -309 ms | | ◆ Search -309 ms; … | -| ↳ transform | +899 us | | ◆ Apply +641 us; … | -| ↳ typing | +699 us | | ◆ Apply +437 us; … | -| ↳ misc | +6.33 us | | ◆ Assembly +6.33 us | -| Equality/rebuild — net | +1.61 ms | -0.552% | ◆ Assembly +1.21 ms; … | -| ↳ @rebuilding | +1.43 ms | | ◆ Assembly +1.02 ms; … | -| ↳ @parent | +187 us | | ◆ Assembly +178 us; … | -| ↳ @rebuilding_cleanup | +1.31 us | | ◆ Assembly +1.31 us | -| ↳ @subsume_ruleset | +1.22 us | | ◆ Assembly +1.22 us | +| Program rules — own work | -303 ms | +105% | ◆ Search -305 ms; … | +| ↳ mapping | -305 ms | | ◆ Search -305 ms; … | +| ↳ transform | +923 us | | ◆ Apply +668 us; … | +| ↳ typing | +741 us | | ◆ Apply +466 us; … | +| ↳ misc | +7.69 us | | ◆ Assembly +7.69 us | +| Equality/rebuild — net | +1.74 ms | -0.606% | ◆ Assembly +1.30 ms; … | +| ↳ @rebuilding | +1.53 ms | | ◆ Assembly +1.09 ms; … | +| ↳ @parent | +213 us | | ◆ Assembly +203 us; … | +| ↳ @rebuilding_cleanup | +1.48 us | | ◆ Assembly +1.48 us | +| ↳ @subsume_ruleset | +1.10 us | | ◆ Assembly +1.10 us | *Program + Equality account for +105% of this file's wall-time change. Source rules shown: 4/4. Maintenance rules shown: 4/4.* @@ -228,33 +213,33 @@ | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +59.5 ms | +57.9% | ◆ Apply +40.4 ms; Merge +22.8 ms; … | -| ↳ rules | +59.5 ms | | ◆ Apply +40.4 ms; Merge +22.8 ms; … | -| Equality/rebuild — net | +16.8 ms | +16.3% | Assembly +1.73 ms; ◆ Search +15.4 ms; Apply +6.26 ms; Merge +4.05 ms; Rebuild -11.1 ms; … | -| ↳ @rebuilding | +26.9 ms | | ◆ Search +14.6 ms; Apply +6.25 ms; Merge +3.95 ms; … | -| ↳ @parent | +950 us | | ◆ Search +747 us; … | -| ↳ @rebuilding_cleanup | +701 ns | | ◆ Assembly +701 ns | -| ↳ @subsume_ruleset | +334 ns | | ◆ Assembly +334 ns | -| ↳ Native rebuild replaced | -11.1 ms | | ◆ Rebuild -11.1 ms | +| Program rules — own work | +60.5 ms | +57.8% | ◆ Apply +40.8 ms; Merge +23.5 ms; … | +| ↳ rules | +60.5 ms | | ◆ Apply +40.8 ms; Merge +23.5 ms; … | +| Equality/rebuild — net | +17.1 ms | +16.3% | Assembly +1.93 ms; ◆ Search +15.6 ms; Apply +6.35 ms; Merge +4.13 ms; Rebuild -11.4 ms; … | +| ↳ @rebuilding | +27.5 ms | | ◆ Search +14.9 ms; Apply +6.33 ms; Merge +4.03 ms; … | +| ↳ @parent | +996 us | | ◆ Search +776 us; … | +| ↳ @rebuilding_cleanup | +784 ns | | ◆ Assembly +784 ns | +| ↳ @subsume_ruleset | +319 ns | | ◆ Assembly +319 ns | +| ↳ Native rebuild replaced | -11.4 ms | | ◆ Rebuild -11.4 ms | -*Program + Equality account for +74.2% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* +*Program + Equality account for +74.1% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* ### Ruleset drivers — speq-preserved-reference-suite.egg | Driver | Δ | Wall share | Important phase changes | | --- | ---: | ---: | --- | -| Program rules — own work | +88.2 ms | +74.5% | ◆ Assembly +71.8 ms; Search +9.67 ms; … | -| ↳ parseIR.transform-taco-spmv-csc | +27.8 ms | | ◆ Assembly +19.7 ms; Search +4.84 ms; Execution +3.23 ms; … | -| ↳ parseIR.transform-csparse-spmv-csc-nostruct | +27.4 ms | | ◆ Assembly +19.4 ms; Search +4.74 ms; Execution +3.20 ms; … | -| ↳ parseIR.transform-parboil-hist | +16.5 ms | | ◆ Assembly +16.3 ms; … | -| ↳ parseIR.transform-npb-is-hist | +16.3 ms | | ◆ Assembly +16.2 ms; … | -| ↳ parseIR.expand-parboil-hist | +49.8 us | | ◆ Assembly +25.3 us; … | -| ↳ Other (3 more source rulesets) | +117 us | | ◆ Assembly +73.8 us; … | -| Equality/rebuild — net | +782 us | +0.660% | ◆ Assembly +478 us; … | -| ↳ @rebuilding | +783 us | | ◆ Assembly +440 us; … | -| ↳ @parent | +53.3 us | | ◆ Assembly +35.5 us; … | -| ↳ @subsume_ruleset | +1.27 us | | ◆ Assembly +1.27 us | -| ↳ @rebuilding_cleanup | +1.26 us | | ◆ Assembly +1.26 us | -| ↳ Native rebuild replaced | -56.5 us | | ◆ Rebuild -56.5 us | - -*Program + Equality account for +75.1% of this file's wall-time change. Source rules shown: 5/8 plus exact Other. Maintenance rules shown: 4/4.* +| Program rules — own work | +90.4 ms | +73.9% | ◆ Assembly +73.5 ms; Search +9.92 ms; … | +| ↳ parseIR.transform-taco-spmv-csc | +28.5 ms | | ◆ Assembly +20.2 ms; Search +4.92 ms; Execution +3.28 ms; … | +| ↳ parseIR.transform-csparse-spmv-csc-nostruct | +28.1 ms | | ◆ Assembly +19.9 ms; Search +4.88 ms; Execution +3.26 ms; … | +| ↳ parseIR.transform-parboil-hist | +17.0 ms | | ◆ Assembly +16.7 ms; … | +| ↳ parseIR.transform-npb-is-hist | +16.7 ms | | ◆ Assembly +16.6 ms; … | +| ↳ parseIR.expand-parboil-hist | +52.7 us | | ◆ Assembly +29.4 us; … | +| ↳ Other (3 more source rulesets) | +132 us | | ◆ Assembly +81.7 us; … | +| Equality/rebuild — net | +846 us | +0.691% | ◆ Assembly +520 us; … | +| ↳ @rebuilding | +844 us | | ◆ Assembly +477 us; … | +| ↳ @parent | +59.9 us | | ◆ Assembly +41.1 us; … | +| ↳ @rebuilding_cleanup | +1.27 us | | ◆ Assembly +1.27 us | +| ↳ @subsume_ruleset | +1.10 us | | ◆ Assembly +1.10 us | +| ↳ Native rebuild replaced | -61.0 us | | ◆ Rebuild -61.0 us | + +*Program + Equality account for +74.6% of this file's wall-time change. Source rules shown: 5/8 plus exact Other. Maintenance rules shown: 4/4.* diff --git a/term-encoding-overhead-breakdown.md b/term-encoding-overhead-breakdown.md index acabe141..45bf6331 100644 --- a/term-encoding-overhead-breakdown.md +++ b/term-encoding-overhead-breakdown.md @@ -2,29 +2,29 @@ ## Result -Term encoding is `1.61–1.63x` slower across the current ten-workload suite, but -there is no single dominant cause. The suite mean adds 1.889 seconds over a -3.035-second `off` baseline: +Term encoding is `1.62–1.64x` slower across the current ten-workload suite, but +there is no single dominant cause. The suite mean adds 1.935 seconds over a +3.083-second `off` baseline: | Mechanism | Mean delta | Share of slowdown | | --- | ---: | ---: | -| Source-rule execution | +563 ms | 29.8% | -| Equality/rebuild, net | +455 ms | 24.1% | -| Typechecking | +345 ms | 18.2% | -| Other frontend/install | +344 ms | 18.2% | -| Commands | +136 ms | 7.22% | -| Residual | +46.4 ms | 2.46% | +| Source-rule execution | +575 ms | 29.7% | +| Equality/rebuild, net | +476 ms | 24.6% | +| Typechecking | +350 ms | 18.1% | +| Other frontend/install | +346 ms | 17.9% | +| Commands | +138 ms | 7.12% | +| Residual | +50.3 ms | 2.60% | The full generated report is checked in as [`term-encoding-overhead-benchmark.md`](term-encoding-overhead-benchmark.md). The important engineering conclusion is that a native or inline rebuild alone cannot reach the 5–10% target. Making the entire Equality/rebuild bucket as -cheap as the baseline would reduce the suite point ratio only from `1.62x` to +cheap as the baseline would reduce the suite point ratio only from `1.63x` to `1.47x`. Removing all measured non-program overhead would still leave `1.20x` because transformed source rules remain materially different. Reaching `1.10x` would require removing about 84% of the current added time, including roughly -306 ms, or 54%, of the net Program bucket even after every positive non-program +317 ms, or 55%, of the net Program bucket even after every positive non-program delta disappeared. ## Measurement @@ -35,9 +35,10 @@ commit `d60202f64424`; the later report-only commit does not change that binary. ```bash ./bench.py \ + --target @d60202f \ + --compare-target @d60202f \ --detail rulesets \ --treatment term \ - --force-run \ --report /tmp/term-encoding-d60202f.jsonl \ --format markdown ``` @@ -52,19 +53,19 @@ The actual wall-time ratios were: | Workload | `term / off` (95% CI) | | --- | ---: | -| Math | 2.01–2.11x | -| eggcc | 1.43–1.48x | -| Pointer analysis | 2.27–2.37x | -| Hardboiled | 1.88–1.94x | -| Luminal | 3.38–3.46x | -| Herbie | 1.96–2.02x | -| Misaal HVX | 2.95–3.07x | -| Churchroad wide multiply | 0.705–0.715x | -| DialEgg NMM40 | 1.64–1.66x | -| SPEQ preserved-reference suite | 3.56–3.61x | +| Math | 1.98–2.08x | +| eggcc | 1.45–1.48x | +| Pointer analysis | 2.27–2.35x | +| Hardboiled | 1.91–1.95x | +| Luminal | 3.39–3.43x | +| Herbie | 1.95–2.03x | +| Misaal HVX | 2.97–3.05x | +| Churchroad wide multiply | 0.708–0.720x | +| DialEgg NMM40 | 1.61–1.69x | +| SPEQ preserved-reference suite | 3.55–3.65x | Churchroad is a useful warning against treating every encoding-induced change -as overhead: its `mapping` Search becomes 309 ms faster, more than offsetting +as overhead: its `mapping` Search becomes 305 ms faster, more than offsetting the added frontend and maintenance work. ### Timer-tax control @@ -80,93 +81,95 @@ also same-binary, so both of its endpoints pay the retained instrumentation. ## What “inline rebuilding” can mean -The report separates two distinct counterfactuals: +The measured leaves support two distinct interpretations of “inline”: -1. **Remove Equality ruleset assembly.** This removes 299 ms of lazy plan +1. **Remove Equality ruleset assembly.** This removes 307 ms of lazy plan creation and per-invocation executable-ruleset construction, producing an - implied `1.52x` suite ratio. + implied `1.53x` suite ratio. 2. **Make net Equality/rebuild baseline-equivalent.** This removes the entire - 455 ms net responsibility, producing an implied `1.47x` ratio. + 476 ms net responsibility, producing an implied `1.47x` ratio. The second number is the optimistic answer to “what if the relational UF and rebuild were as cheap as native rebuilding?” It is not the cost of one named ruleset. Encoded maintenance is collective: `@rebuilding`, `@parent`, cleanup, and subsumption together replace the native rebuild loop. -Across the suite, generated Equality maintenance adds 863 ms before crediting -the 407 ms of native Rebuild it replaces: +Across the suite, generated Equality maintenance adds 894 ms before crediting +the 417 ms of native Rebuild it replaces: | Equality phase | Mean delta | | --- | ---: | -| Assembly | +299 ms | -| Search | +375 ms | -| Apply | +77.8 ms | -| Execution | +13.2 ms | -| Merge | +96.9 ms | -| Native Rebuild replaced | −407 ms | -| **Net Equality/rebuild** | **+455 ms** | +| Assembly | +307 ms | +| Search | +391 ms | +| Apply | +80.5 ms | +| Execution | +14.0 ms | +| Merge | +101 ms | +| Native Rebuild replaced | −417 ms | +| **Net Equality/rebuild** | **+476 ms** | So a plan-cache or inline-assembly change attacks a real cost, especially on eggcc, but it leaves most Equality Search/Apply/Merge work intact. Conversely, folding the native-rebuild credit into `@rebuilding` would falsely make that single generated ruleset look cheap and obscure the collective substitution. -## Optimization ceilings +## Derived bounds for the engineering question -The report now performs the arithmetic directly. Each row removes only the -named positive candidate-minus-baseline deltas, preserves candidate-side -speedups, and holds every other mean fixed. +The suite row is a sum of per-file mean deltas, not one process observation. +Its additive cells support simple what-if arithmetic, but that arithmetic is +deliberately not another report table: the combinations are editorial, add no +measurement, and hide that a mechanism can dominate one workload while being +irrelevant to another. -| Hypothetical change | Time removed | Implied ratio | -| --- | ---: | ---: | -| Remove added typechecking | 345 ms | 1.51x | -| Remove added frontend/install | 344 ms | 1.51x | -| Remove both frontend groups | 689 ms | 1.40x | -| Remove Equality assembly | 299 ms | 1.52x | -| Remove net Equality/rebuild | 455 ms | 1.47x | -| Remove source-rule execution delta | 563 ms | 1.44x | -| Remove every positive non-program delta | 1.28 s | 1.20x | -| Remove every recorded positive mechanism delta | 1.84 s | 1.02x | - -These are additive accounting ceilings, not implementation predictions. They -have no confidence intervals and do not model interactions: removing generated -types or identities may also change Program Search, Apply, Merge, or plan -assembly. The `1.02x` final row is primarily an accounting-closure check; it -leaves the 46 ms residual rather than pretending uninstrumented time is freely -removable. +For the specific always-on design question: + +- matching the baseline's typechecking cost alone implies `1.51x`; matching + other frontend/install alone implies `1.52x`, and matching both implies + `1.40x`; +- eliminating Equality ruleset assembly alone implies `1.53x`, while making + the entire net Equality/rebuild responsibility baseline-equivalent implies + `1.47x`; +- eliminating the net source-rule execution delta implies `1.44x`; and +- even eliminating every positive non-program delta while holding the Program + delta fixed implies about `1.20x`. + +These are point-estimate accounting bounds, not implementation predictions. +They have no confidence intervals and do not model interactions: removing +generated types or identities may also change Program Search, Apply, Merge, or +plan assembly. The per-workload rows below are the primary evidence for +choosing an optimization. ## Workload narratives -- **Math:** Equality/rebuild is 60.6% of the slowdown. Generated maintenance - costs 395 ms and replaces 135 ms of native rebuild. This is the clearest +- **Math:** Equality/rebuild is 62.5% of the slowdown. Generated maintenance + costs 412 ms and replaces 142 ms of native rebuild. This is the clearest relational-UF target, though changed default-rule Apply and Merge still add - 152 ms. -- **eggcc:** no single mechanism wins. Typecheck plus frontend adds 136 ms, - source rules add 108 ms, net Equality adds 83 ms, and commands add 35 ms. - Equality is assembly-heavy: 245 ms of added assembly is almost exactly - offset by 245 ms of removed native rebuild before Search and execution are - counted. `always-run` carries 82 ms of the Program delta. -- **Pointer analysis:** net Equality is largest at 29 ms, frontend is 21 ms, - and Program is 11 ms. Its 9.5 ms residual is large enough that tiny + 145 ms. +- **eggcc:** no single mechanism wins. Typecheck plus frontend adds 139 ms, + source rules add 113 ms, net Equality adds 86 ms, and commands add 35 ms. + Equality is assembly-heavy: 248 ms of added assembly is almost exactly + offset by 247 ms of removed native rebuild before Search and execution are + counted. `always-run` carries 86 ms of the Program delta. +- **Pointer analysis:** net Equality is largest at 30 ms, frontend is 21 ms, + and Program is 11 ms. Its 10.1 ms residual is large enough that tiny sub-mechanism conclusions should remain cautious. -- **Hardboiled:** source rules add 33 ms, while typecheck plus frontend adds +- **Hardboiled:** source rules add 35 ms, while typecheck plus frontend adds 49 ms. The default ruleset's Search dominates its Program child; check evaluation is routed symmetrically under Commands rather than appearing as a term-only default ruleset artifact. -- **Luminal:** Program is 403 ms, 46.5% of the slowdown; Equality is only - 39.8 ms, 4.59%. `fusion_grow` and `fusion_pair` add 169 and 147 ms, almost +- **Luminal:** Program is 408 ms, 46.4% of the slowdown; Equality is only + 44.8 ms, 5.10%. `fusion_grow` and `fusion_pair` add 170 and 148 ms, almost entirely Search. Typecheck plus frontend adds another 344 ms. UF work is not the limiting explanation here. -- **Herbie:** mixed across Program (26.6%), Equality (22.4%), frontend/typecheck - (33.1%), and Commands (14.8%). -- **Misaal HVX:** typecheck plus frontend explains 90.6% of the slowdown. - Program and Equality together explain less than 4%; a UF optimization would +- **Herbie:** mixed across Program (26.6%), Equality (22.3%), frontend/typecheck + (32.8%), and Commands (15.4%). +- **Misaal HVX:** typecheck plus frontend explains 90.3% of the slowdown. + Program and Equality together explain about 4.1%; a UF optimization would barely move it. -- **Churchroad:** Program Search improves by 308 ms net, making term encoding +- **Churchroad:** Program Search improves by 303 ms net, making term encoding faster overall despite every other top-level mechanism becoming slower. -- **DialEgg:** Program contributes 57.9%, led by Apply and Merge; net Equality +- **DialEgg:** Program contributes 57.8%, led by Apply and Merge; net Equality contributes 16.3%. -- **SPEQ:** Program contributes 74.5%, mostly assembly in four transform +- **SPEQ:** Program contributes 73.9%, mostly assembly in four transform rulesets; Equality is below 1%. ## What this answers—and what it does not @@ -177,13 +180,13 @@ The additive report now answers: equality/rebuild, commands, or residual; - whether Equality cost is assembly or execution; - which source or maintenance rulesets carry Program and Equality changes; and -- optimistic remaining ratios when selected positive deltas disappear. +- the suite sum and the distinct per-workload mechanism mixes. It does not identify why a source rule searches or assembles more slowly. For Luminal, the data localizes the problem to `fusion_grow`/`fusion_pair` Search, but distinguishing wider tuples, extra identity columns, changed join order, or greater state churn requires a profiler or a targeted lowering ablation. -Likewise, the counterfactual rows cannot predict cross-mechanism effects. +Likewise, the derived bounds cannot predict cross-mechanism effects. ## Measurement design @@ -226,17 +229,17 @@ The retained complexity has five responsibilities: | Engine timing | Exclusive process and six-phase ruleset boundaries | Static path slices and one duration map | | Semantic routing | Program, maintenance, native rebuild, and check ownership | One two-variant role instead of name-prefix inference | | Transport | Persist exact leaves for later projections | One open segmented-path list; no fixed phase structs or second ruleset schema | -| Analysis | Align endpoint samples, derive residual, mechanisms, drivers, and ceilings | One generic path-sample ledger; parent/child sums are direct | -| Presentation | Decomposition, suite ceilings, and per-file drivers | One shared catalog and table renderer for Rich, Markdown, and interactive output | +| Analysis | Align endpoint samples and derive residual, mechanisms, and drivers | One generic path-sample ledger; parent/child sums are direct | +| Presentation | Decomposition and per-file drivers | One shared catalog and table renderer for Rich, Markdown, and interactive output | The final reduction pass kept the old execution path recognizable and removed presentation-only alternatives: there is no global phase-rollup model, no ten-column ruleset table, no duplicated fixed five-bucket wire record, and no -special report-time native-rebuild credit. The optimization table is derived -from the same means and leaf ledger rather than introducing another recording -shape. Further reduction would either lose the Assembly/Search distinction -that separates plan-cache work from query-shape work or make the signs in the -ruleset panel misleading again. +special report-time native-rebuild credit. Counterfactual combinations stay in +the engineering analysis rather than becoming another report model. Further +reduction would either lose the Assembly/Search distinction that separates +plan-cache work from query-shape work or make the signs in the ruleset panel +misleading again. ## Engineering direction diff --git a/tests/__snapshots__/test_report_rendering.ambr b/tests/__snapshots__/test_report_rendering.ambr index c73d43dc..c1420036 100644 --- a/tests/__snapshots__/test_report_rendering.ambr +++ b/tests/__snapshots__/test_report_rendering.ambr @@ -46,26 +46,11 @@ | File | Wall Δ | Typecheck | Frontend | Program | Equality | Commands | Residual | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | - | Suite total | +200 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% +116 ms | 0% 0 ms | 0% 0 ms | +42.0% +84.0 ms | + | Suite total (2 files) | +200 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% +116 ms | 0% 0 ms | 0% 0 ms | +42.0% +84.0 ms | | math.egg | -200 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% -116 ms | 0% 0 ms | 0% 0 ms | +42.0% -84.0 ms | | rewrite.egg | +400 ms | 0% 0 ms | 0% 0 ms | ◆ +58.0% +232 ms | 0% 0 ms | 0% 0 ms | +42.0% +168 ms | - *Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* - - ### Optimization ceilings - - | Hypothetical change | Time removed | Remaining wall Δ | Implied ratio | - | --- | ---: | ---: | ---: | - | Remove added typechecking time | 0 ns | +200 ms | 1.07x | - | Remove added frontend/install time | 0 ns | +200 ms | 1.07x | - | Remove added typechecking + frontend time | 0 ns | +200 ms | 1.07x | - | Remove added Equality assembly time | 0 ns | +200 ms | 1.07x | - | Remove added net Equality/rebuild time | 0 ns | +200 ms | 1.07x | - | Remove added source-rule execution time | +116 ms | +84.0 ms | 1.03x | - | Remove every added non-program mechanism | 0 ns | +200 ms | 1.07x | - | Remove every recorded added mechanism | +116 ms | +84.0 ms | 1.03x | - - *Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is never removed.* + *The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. Each mechanism cell is its share of that row's wall-time change followed by its signed mean time change. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* ## Ruleset drivers @@ -204,7 +189,8 @@ File Wall Δ Typecheck Frontend Program Equality Commands Residual ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Suite total +1550 ms 0% 0 ms 0% 0 ms +10.7% +165 ms 0% 0 ms 0% 0 ms ◆ +89.3% +1385 ms + Suite total (6 +1550 ms 0% 0 ms 0% 0 ms +10.7% +165 ms 0% 0 ms 0% 0 ms ◆ +89.3% +1385 ms + files) math.egg -200 ms 0% 0 ms 0% 0 ms +14.6% -29.1 ms +0.985% -1.97 ms 0% 0 ms ◆ +84.4% -169 ms eggcc-extract.egg -150 ms 0% 0 ms 0% 0 ms +21.1% -31.7 ms +0.788% -1.18 ms 0% 0 ms ◆ +78.1% -117 ms pointer-analysis- 0 ms — 0 ms — 0 ms — -15.3 ms — -0.394 ms — 0 ms — +15.7 ms @@ -213,31 +199,14 @@ luminal.egg +600 ms 0% 0 ms 0% 0 ms +12.4% +74.2 ms +0.197% +1.18 ms 0% 0 ms ◆ +87.4% +525 ms herbie.egg +1050 ms 0% 0 ms 0% 0 ms +14.0% +147 ms +0.188% +1.97 ms 0% 0 ms ◆ +85.8% +901 ms - Each mechanism cell is its share of the wall-time slowdown followed by candidate − baseline mean time. Frontend includes + The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. Each + mechanism cell is its share of that row's wall-time change followed by its signed mean time change. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative. - Optimization ceilings - - Hypothetical change Time removed Remaining wall Δ Implied ratio - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Remove added typechecking time 0 ns +1550 ms 1.11x - Remove added frontend/install time 0 ns +1550 ms 1.11x - Remove added typechecking + frontend time 0 ns +1550 ms 1.11x - Remove added Equality assembly time 0 ns +1550 ms 1.11x - Remove added net Equality/rebuild time 0 ns +1550 ms 1.11x - Remove added source-rule execution time +165 ms +1385 ms 1.10x - Remove every added non-program mechanism 0 ns +1550 ms 1.11x - Remove every recorded added mechanism +165 ms +1385 ms 1.10x - - Each row mechanically removes the named positive candidate − baseline timing deltas while holding every other measured - mean fixed; candidate advantages are retained. Equality assembly removes only ruleset assembly, while net - Equality/rebuild removes the whole positive Equality bucket. These are optimistic additive accounting bounds, not - implementation predictions; point ratios have no confidence intervals and interactions can invalidate them. Residual is - never removed. ─────────────────────────────────────────────────── Per-file results ─────────────────────────────────────────────────── Wall time diff --git a/tests/test_report_analysis.py b/tests/test_report_analysis.py index 9d4ce99a..7e91e522 100644 --- a/tests/test_report_analysis.py +++ b/tests/test_report_analysis.py @@ -371,84 +371,6 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp assert native_rebuild.delta == equality.delta -def test_optimization_ceilings_reset_suite_deltas_without_claiming_implementation_speedups( - tmp_path: Path, -) -> None: - report = tmp_path / "report.jsonl" - comparison = _comparison(tmp_path) - candidate_timing = make_timing_summary( - make_ruleset_timing( - assembly_ns=31, - search_ns=37, - apply_ns=41, - execution_ns=43, - merge_ns=47, - rebuild_ns=53, - ), - make_ruleset_timing( - name="@rebuilding", - role="equality", - assembly_ns=61, - search_ns=0, - apply_ns=0, - execution_ns=0, - merge_ns=0, - rebuild_ns=0, - ), - typecheck_ns=13, - frontend_parse_ns=11, - frontend_other_ns=17, - frontend_install_ns=19, - commands_actions_ns=23, - commands_check_ns=7, - commands_other_ns=29, - ) - write_report( - report, - make_record( - 0, - started_at="2026-07-15T12:00:00Z", - binary_sha256="sha256:baseline", - wall_sec=0.000001, - timing_summary=make_timing_summary( - make_ruleset_timing( - assembly_ns=0, - search_ns=0, - apply_ns=0, - execution_ns=0, - merge_ns=0, - rebuild_ns=0, - ) - ), - ), - make_record( - 1, - started_at="2026-07-15T12:00:01Z", - binary_sha256="sha256:candidate", - wall_sec=0.0000015, - timing_summary=candidate_timing, - ), - ) - - ceilings = analyze_pair(ReportStore(report), comparison, "phases").ceilings - - assert [row.scenario for row in ceilings] == [ - "typecheck", - "frontend", - "frontend_and_typecheck", - "equality_assembly", - "equality", - "program", - "non_program", - "all_recorded", - ] - assert [row.reset_delta_ns for row in ceilings] == pytest.approx([13, 47, 60, 61, 114, 199, 233, 432]) - assert [row.remaining_delta_ns for row in ceilings] == pytest.approx([487, 453, 440, 439, 386, 301, 267, 68]) - assert [row.counterfactual_ratio for row in ceilings] == pytest.approx( - [1.487, 1.453, 1.440, 1.439, 1.386, 1.301, 1.267, 1.068] - ) - - @pytest.mark.parametrize( ("path", "message"), ((["residual", "stored"], "residual is derived"), (["mystery", "work"], "unknown timing responsibility")), diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index 1fa29403..2e56a257 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -229,14 +229,14 @@ def test_all_rich_tables_use_one_compact_style(tmp_path: Path) -> None: assert all(table.box is box.SIMPLE_HEAVY and not table.show_lines for table in tables) -def test_phase_detail_adds_compact_suite_optimization_ceilings(tmp_path: Path) -> None: +def test_phase_detail_is_one_additive_decomposition_table(tmp_path: Path) -> None: report_path, comparison = _pair_case(tmp_path) catalog = build_report_catalog(ReportStore(report_path), comparison, "phases") section = next(section for section in catalog.sections if section.id == "phases") tables = tuple(block for block in section.blocks if isinstance(block, ReportTable)) - assert len(tables) == 2 - table, ceilings = tables + assert len(tables) == 1 + (table,) = tables assert tuple(column.id for column in table.columns) == ( "file", "wall_delta", @@ -248,7 +248,7 @@ def test_phase_detail_adds_compact_suite_optimization_ceilings(tmp_path: Path) - "residual", ) assert len(table.rows) == len(comparison.files) + 1 - assert table.rows[0].cells[0].display == "Suite total" + assert table.rows[0].cells[0].display == "Suite total (2 files)" assert [row.cells[0].display for row in table.rows[1:]] == ["math.egg", "rewrite.egg"] assert table.columns[3].label == "Frontend" assert table.columns[4].label == "Program" @@ -261,26 +261,6 @@ def test_phase_detail_adds_compact_suite_optimization_ceilings(tmp_path: Path) - assert table.rows[1].cells[1].tone == "positive" assert table.rows[1].cells[4].tone == "emphasis" assert table.rows[1].cells[7].tone == "positive" - assert ceilings.title == "Optimization ceilings" - assert tuple(column.id for column in ceilings.columns) == ( - "scenario", - "reset_delta", - "remaining_delta", - "counterfactual_ratio", - ) - assert [row.cells[0].display for row in ceilings.rows] == [ - "Remove added typechecking time", - "Remove added frontend/install time", - "Remove added typechecking + frontend time", - "Remove added Equality assembly time", - "Remove added net Equality/rebuild time", - "Remove added source-rule execution time", - "Remove every added non-program mechanism", - "Remove every recorded added mechanism", - ] - assert all(row.cells[3].display.endswith("x") for row in ceilings.rows) - assert ceilings.caption is not None and "accounting bounds, not implementation predictions" in ceilings.caption - assert "Residual is never removed" in ceilings.caption def test_ruleset_detail_unfolds_program_and_equality_with_explicit_children(tmp_path: Path) -> None: From b3ee888a2438bc76c209e9bd62a6c969e10010fe Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 00:16:41 -0400 Subject: [PATCH 5/9] Align repeated report table columns --- README.md | 4 + benchmarking/reports/render.py | 76 ++++++++- .../__snapshots__/test_report_rendering.ambr | 148 +++++++++--------- tests/test_report_rendering.py | 24 +++ 4 files changed, 173 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 9e890706..c018efe3 100644 --- a/README.md +++ b/README.md @@ -416,6 +416,10 @@ with `◆`, and uses fixed Assembly, Search, Apply, Execution, Merge, Rebuild order. `…` means smaller nonzero phase changes were omitted from display, not from accounting. +Rich gives every repeated table schema one content-derived column layout, so +wall/RSS results align with each other and all per-file driver panels retain +the same scan positions. Markdown remains width-independent. + Benchmarks run single-threaded. This keeps Search and Apply attribution additive for egglog's interleaved executor. diff --git a/benchmarking/reports/render.py b/benchmarking/reports/render.py index f43129f8..b1a7532e 100644 --- a/benchmarking/reports/render.py +++ b/benchmarking/reports/render.py @@ -9,6 +9,7 @@ from __future__ import annotations from rich import box +from rich.cells import cell_len from rich.console import Group, RenderableType from rich.rule import Rule from rich.table import Table @@ -17,6 +18,7 @@ from .catalog import CellTone, ReportCatalog, ReportMessage, ReportSection, ReportTable RICH_DETAIL_MIN_WIDTH = 120 +RICH_TEXT_COLUMN_MIN_WIDTH = 12 RICH_DETAIL_NARROW_WARNING = ( "Warning: detailed Rich report output is designed for terminals at least 120 columns wide " "(detected {width}); output may wrap. Widen the terminal or use --format markdown." @@ -50,18 +52,43 @@ def report_table(title: str | None, *, caption: str | None = None) -> Table: ) -def render_rich_table(table_data: ReportTable, *, show_title: bool = True) -> Table: +def render_rich_table( + table_data: ReportTable, + *, + show_title: bool = True, + preferred_widths: tuple[int, ...] | None = None, +) -> Table: """Render one catalog table without interpreting its display strings.""" table = report_table( table_data.title if show_title else None, caption=table_data.caption, ) - for column in table_data.columns: + widths: tuple[int | None, ...] = preferred_widths or tuple(None for _ in table_data.columns) + for column, preferred_width in zip(table_data.columns, widths, strict=True): + if preferred_width is None: + ratio = width = min_width = None + no_wrap = False + elif column.alignment == "right": + ratio = min_width = None + width = preferred_width + no_wrap = True + else: + ratio = preferred_width + width = None + min_width = min( + preferred_width, + max(_max_line_width(column.label), RICH_TEXT_COLUMN_MIN_WIDTH), + ) + no_wrap = False table.add_column( Text(column.label), justify="right" if column.alignment == "right" else "left", overflow="fold", + ratio=ratio, + width=width, + min_width=min_width, + no_wrap=no_wrap, ) for row in table_data.rows: table.add_row(*(Text(cell.display, style=TONE_STYLES[cell.tone]) for cell in row.cells)) @@ -95,6 +122,7 @@ def render_markdown_table(table_data: ReportTable, *, heading_level: int | None def render_rich_report_document(catalog: ReportCatalog, width: int) -> Group: """Render rulesets, phases, files, comparison, then the final summary.""" + shared_column_widths = _shared_column_widths(catalog) sections = {section.id: section for section in catalog.sections} ordered = tuple(sections[section_id] for section_id in RICH_SECTION_ORDER if section_id in sections) ordered_ids = {section.id for section in ordered} @@ -106,7 +134,7 @@ def render_rich_report_document(catalog: ReportCatalog, width: int) -> Group: if warning_pending and section.id in DETAIL_SECTION_IDS: renderables.append(Text(RICH_DETAIL_NARROW_WARNING.format(width=width), style="yellow")) warning_pending = False - renderables.extend(_rich_section_renderables(section)) + renderables.extend(_rich_section_renderables(section, shared_column_widths)) return Group(*renderables) @@ -119,7 +147,10 @@ def render_markdown_report_document(catalog: ReportCatalog) -> str: return "\n\n".join(part.strip() for part in parts if part.strip()) -def _rich_section_renderables(section: ReportSection) -> tuple[RenderableType, ...]: +def _rich_section_renderables( + section: ReportSection, + shared_column_widths: dict[str, tuple[int, ...]], +) -> tuple[RenderableType, ...]: renderables: list[RenderableType] = [] if section.title is not None: renderables.append(Rule(Text(section.title, style="bold"), style="green")) @@ -129,6 +160,7 @@ def _rich_section_renderables(section: ReportSection) -> tuple[RenderableType, . _render_rich_block( block, show_table_title=not (index == 0 and hide_first_table_title), + preferred_widths=shared_column_widths.get(block.id) if isinstance(block, ReportTable) else None, ) ) return tuple(renderables) @@ -159,9 +191,10 @@ def _render_rich_block( block: ReportTable | ReportMessage, *, show_table_title: bool = True, + preferred_widths: tuple[int, ...] | None = None, ) -> RenderableType: if isinstance(block, ReportTable): - return render_rich_table(block, show_title=show_table_title) + return render_rich_table(block, show_title=show_table_title, preferred_widths=preferred_widths) if block.title is None: return Text(block.text, style=TONE_STYLES[block.tone] or "dim") return Group(Text(block.title, style="bold"), Text(block.text, style=TONE_STYLES[block.tone] or "dim")) @@ -177,3 +210,36 @@ def _render_markdown_block(block: ReportTable | ReportMessage) -> str: def _markdown_heading(value: str) -> str: return value.replace("\n", " ").replace("#", "\\#") + + +def _shared_column_widths(catalog: ReportCatalog) -> dict[str, tuple[int, ...]]: + """Give repeated table schemas one content-derived Rich column layout.""" + + families: dict[tuple[tuple[str, str, str], ...], list[ReportTable]] = {} + for section in catalog.sections: + for block in section.blocks: + if isinstance(block, ReportTable): + signature = tuple((column.id, column.label, column.alignment) for column in block.columns) + families.setdefault(signature, []).append(block) + + result: dict[str, tuple[int, ...]] = {} + for tables in families.values(): + if len(tables) < 2: + continue + widths = tuple( + max( + 1, + _max_line_width(tables[0].columns[column_index].label), + *(_max_line_width(row.cells[column_index].display) for table in tables for row in table.rows), + ) + for column_index in range(len(tables[0].columns)) + ) + for table in tables: + result[table.id] = widths + return result + + +def _max_line_width(value: str) -> int: + """Measure the widest terminal line without counting Unicode code points as cells.""" + + return max((cell_len(line) for line in value.splitlines()), default=0) diff --git a/tests/__snapshots__/test_report_rendering.ambr b/tests/__snapshots__/test_report_rendering.ambr index c1420036..2b0ef8b9 100644 --- a/tests/__snapshots__/test_report_rendering.ambr +++ b/tests/__snapshots__/test_report_rendering.ambr @@ -91,97 +91,97 @@ omitted nonzero phases. Ruleset drivers — math.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work -29.1 ms +14.6% ◆ Search -15.8 ms; Apply -7.88 ms; Merge -3.94 ms; … - ↳ ruleset-11 -4.48 ms ◆ Search -2.42 ms; Apply -1.21 ms; … - ↳ ruleset-10 -4.11 ms ◆ Search -2.22 ms; Apply -1.11 ms; … - ↳ ruleset-09 -3.74 ms ◆ Search -2.02 ms; Apply -1.01 ms; … - ↳ ruleset-08 -3.36 ms ◆ Search -1.82 ms; … - ↳ ruleset-07 -2.99 ms ◆ Search -1.62 ms; … - ↳ Other (7 more source rulesets) -10.5 ms ◆ Search -5.66 ms; Apply -2.83 ms; Merge -1.41 ms; … - Equality/rebuild — net -1.97 ms +0.985% ◆ Rebuild -1.97 ms - ↳ Native rebuild replaced -1.97 ms ◆ Rebuild -1.97 ms + Program rules — own work -29.1 ms +14.6% ◆ Search -15.8 ms; Apply -7.88 ms; Merge -3.94 ms; … + ↳ ruleset-11 -4.48 ms ◆ Search -2.42 ms; Apply -1.21 ms; … + ↳ ruleset-10 -4.11 ms ◆ Search -2.22 ms; Apply -1.11 ms; … + ↳ ruleset-09 -3.74 ms ◆ Search -2.02 ms; Apply -1.01 ms; … + ↳ ruleset-08 -3.36 ms ◆ Search -1.82 ms; … + ↳ ruleset-07 -2.99 ms ◆ Search -1.62 ms; … + ↳ Other (7 more source rulesets) -10.5 ms ◆ Search -5.66 ms; Apply -2.83 ms; Merge -1.41 ms; … + Equality/rebuild — net -1.97 ms +0.985% ◆ Rebuild -1.97 ms + ↳ Native rebuild replaced -1.97 ms ◆ Rebuild -1.97 ms Program + Equality account for +15.6% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. Ruleset drivers — eggcc-extract.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work -31.7 ms +21.1% ◆ Search -18.9 ms; Apply -9.45 ms; … - ↳ ruleset-11 -4.87 ms ◆ Search -2.91 ms; Apply -1.45 ms; … - ↳ ruleset-10 -4.47 ms ◆ Search -2.67 ms; Apply -1.33 ms; … - ↳ ruleset-09 -4.06 ms ◆ Search -2.42 ms; Apply -1.21 ms; … - ↳ ruleset-08 -3.65 ms ◆ Search -2.18 ms; Apply -1.09 ms; … - ↳ ruleset-07 -3.25 ms ◆ Search -1.94 ms; … - ↳ Other (7 more source rulesets) -11.4 ms ◆ Search -6.79 ms; Apply -3.39 ms; … - Equality/rebuild — net -1.18 ms +0.788% ◆ Rebuild -1.18 ms - ↳ Native rebuild replaced -1.18 ms ◆ Rebuild -1.18 ms + Program rules — own work -31.7 ms +21.1% ◆ Search -18.9 ms; Apply -9.45 ms; … + ↳ ruleset-11 -4.87 ms ◆ Search -2.91 ms; Apply -1.45 ms; … + ↳ ruleset-10 -4.47 ms ◆ Search -2.67 ms; Apply -1.33 ms; … + ↳ ruleset-09 -4.06 ms ◆ Search -2.42 ms; Apply -1.21 ms; … + ↳ ruleset-08 -3.65 ms ◆ Search -2.18 ms; Apply -1.09 ms; … + ↳ ruleset-07 -3.25 ms ◆ Search -1.94 ms; … + ↳ Other (7 more source rulesets) -11.4 ms ◆ Search -6.79 ms; Apply -3.39 ms; … + Equality/rebuild — net -1.18 ms +0.788% ◆ Rebuild -1.18 ms + ↳ Native rebuild replaced -1.18 ms ◆ Rebuild -1.18 ms Program + Equality account for +21.9% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. Ruleset drivers — pointer-analysis-small.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work -15.3 ms — ◆ Search -9.45 ms; Apply -4.73 ms; … - ↳ ruleset-11 -2.35 ms ◆ Search -1.45 ms; … - ↳ ruleset-10 -2.16 ms ◆ Search -1.33 ms; … - ↳ ruleset-09 -1.96 ms ◆ Search -1.21 ms; … - ↳ ruleset-08 -1.76 ms ◆ Search -1.09 ms; … - ↳ ruleset-07 -1.57 ms ◆ Search -970 us; … - ↳ Other (7 more source rulesets) -5.49 ms ◆ Search -3.39 ms; Apply -1.70 ms; … - Equality/rebuild — net -394 us — ◆ Rebuild -394 us - ↳ Native rebuild replaced -394 us ◆ Rebuild -394 us + Program rules — own work -15.3 ms — ◆ Search -9.45 ms; Apply -4.73 ms; … + ↳ ruleset-11 -2.35 ms ◆ Search -1.45 ms; … + ↳ ruleset-10 -2.16 ms ◆ Search -1.33 ms; … + ↳ ruleset-09 -1.96 ms ◆ Search -1.21 ms; … + ↳ ruleset-08 -1.76 ms ◆ Search -1.09 ms; … + ↳ ruleset-07 -1.57 ms ◆ Search -970 us; … + ↳ Other (7 more source rulesets) -5.49 ms ◆ Search -3.39 ms; Apply -1.70 ms; … + Equality/rebuild — net -394 us — ◆ Rebuild -394 us + ↳ Native rebuild replaced -394 us ◆ Rebuild -394 us Program + Equality coverage is unavailable because wall time did not change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. Ruleset drivers — hardboiled.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work +20.0 ms +8.00% ◆ Search +12.6 ms; Apply +6.30 ms; … - ↳ ruleset-11 +3.08 ms ◆ Search +1.94 ms; … - ↳ ruleset-10 +2.82 ms ◆ Search +1.78 ms; … - ↳ ruleset-09 +2.57 ms ◆ Search +1.62 ms; … - ↳ ruleset-08 +2.31 ms ◆ Search +1.45 ms; … - ↳ ruleset-07 +2.05 ms ◆ Search +1.29 ms; … - ↳ Other (7 more source rulesets) +7.18 ms ◆ Search +4.52 ms; Apply +2.26 ms; … - Equality/rebuild — net +394 us +0.158% ◆ Rebuild +394 us - ↳ Native rebuild replaced +394 us ◆ Rebuild +394 us + Program rules — own work +20.0 ms +8.00% ◆ Search +12.6 ms; Apply +6.30 ms; … + ↳ ruleset-11 +3.08 ms ◆ Search +1.94 ms; … + ↳ ruleset-10 +2.82 ms ◆ Search +1.78 ms; … + ↳ ruleset-09 +2.57 ms ◆ Search +1.62 ms; … + ↳ ruleset-08 +2.31 ms ◆ Search +1.45 ms; … + ↳ ruleset-07 +2.05 ms ◆ Search +1.29 ms; … + ↳ Other (7 more source rulesets) +7.18 ms ◆ Search +4.52 ms; Apply +2.26 ms; … + Equality/rebuild — net +394 us +0.158% ◆ Rebuild +394 us + ↳ Native rebuild replaced +394 us ◆ Rebuild +394 us Program + Equality account for +8.16% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. Ruleset drivers — luminal.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work +74.2 ms +12.4% ◆ Search +47.3 ms; Apply +23.6 ms; … - ↳ ruleset-11 +11.4 ms ◆ Search +7.27 ms; Apply +3.64 ms; … - ↳ ruleset-10 +10.5 ms ◆ Search +6.67 ms; Apply +3.33 ms; … - ↳ ruleset-09 +9.51 ms ◆ Search +6.06 ms; Apply +3.03 ms; … - ↳ ruleset-08 +8.56 ms ◆ Search +5.45 ms; Apply +2.73 ms; … - ↳ ruleset-07 +7.61 ms ◆ Search +4.85 ms; Apply +2.42 ms; … - ↳ Other (7 more source rulesets) +26.6 ms ◆ Search +17.0 ms; Apply +8.48 ms; … - Equality/rebuild — net +1.18 ms +0.197% ◆ Rebuild +1.18 ms - ↳ Native rebuild replaced +1.18 ms ◆ Rebuild +1.18 ms + Program rules — own work +74.2 ms +12.4% ◆ Search +47.3 ms; Apply +23.6 ms; … + ↳ ruleset-11 +11.4 ms ◆ Search +7.27 ms; Apply +3.64 ms; … + ↳ ruleset-10 +10.5 ms ◆ Search +6.67 ms; Apply +3.33 ms; … + ↳ ruleset-09 +9.51 ms ◆ Search +6.06 ms; Apply +3.03 ms; … + ↳ ruleset-08 +8.56 ms ◆ Search +5.45 ms; Apply +2.73 ms; … + ↳ ruleset-07 +7.61 ms ◆ Search +4.85 ms; Apply +2.42 ms; … + ↳ Other (7 more source rulesets) +26.6 ms ◆ Search +17.0 ms; Apply +8.48 ms; … + Equality/rebuild — net +1.18 ms +0.197% ◆ Rebuild +1.18 ms + ↳ Native rebuild replaced +1.18 ms ◆ Rebuild +1.18 ms Program + Equality account for +12.6% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. Ruleset drivers — herbie.egg - Driver Δ Wall share Important phase changes + Driver Δ Wall share Important phase changes ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Program rules — own work +147 ms +14.0% ◆ Search +94.5 ms; Apply +47.3 ms; … - ↳ ruleset-11 +22.7 ms ◆ Search +14.5 ms; Apply +7.27 ms; … - ↳ ruleset-10 +20.8 ms ◆ Search +13.3 ms; Apply +6.67 ms; … - ↳ ruleset-09 +18.9 ms ◆ Search +12.1 ms; Apply +6.06 ms; … - ↳ ruleset-08 +17.0 ms ◆ Search +10.9 ms; Apply +5.45 ms; … - ↳ ruleset-07 +15.1 ms ◆ Search +9.70 ms; Apply +4.85 ms; … - ↳ Other (7 more source rulesets) +52.9 ms ◆ Search +33.9 ms; Apply +17.0 ms; … - Equality/rebuild — net +1.97 ms +0.188% ◆ Rebuild +1.97 ms - ↳ Native rebuild replaced +1.97 ms ◆ Rebuild +1.97 ms + Program rules — own work +147 ms +14.0% ◆ Search +94.5 ms; Apply +47.3 ms; … + ↳ ruleset-11 +22.7 ms ◆ Search +14.5 ms; Apply +7.27 ms; … + ↳ ruleset-10 +20.8 ms ◆ Search +13.3 ms; Apply +6.67 ms; … + ↳ ruleset-09 +18.9 ms ◆ Search +12.1 ms; Apply +6.06 ms; … + ↳ ruleset-08 +17.0 ms ◆ Search +10.9 ms; Apply +5.45 ms; … + ↳ ruleset-07 +15.1 ms ◆ Search +9.70 ms; Apply +4.85 ms; … + ↳ Other (7 more source rulesets) +52.9 ms ◆ Search +33.9 ms; Apply +17.0 ms; … + Equality/rebuild — net +1.97 ms +0.188% ◆ Rebuild +1.97 ms + ↳ Native rebuild replaced +1.97 ms ◆ Rebuild +1.97 ms Program + Equality account for +14.2% of this file's wall-time change. Source rules shown: 5/12 plus exact Other. Maintenance rules shown: none. @@ -210,25 +210,25 @@ ─────────────────────────────────────────────────── Per-file results ─────────────────────────────────────────────────── Wall time - File Baseline (95% CI) Candidate (95% CI) Ratio (95% CI) Result + File Baseline (95% CI) Candidate (95% CI) Ratio (95% CI) Result ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - math.egg 0.941–1.07 s 741–869 ms 0.723–0.885x faster - eggcc-extract.egg 1.44–1.57 s 1.29–1.42 s 0.845–0.959x faster - pointer-analysis-small.egg 1.94–2.07 s 1.94–2.07 s 0.956–1.05x CI includes 1 - hardboiled.egg 2.44–2.57 s 2.69–2.82 s 1.06–1.14x slower - luminal.egg 2.94–3.07 s 3.54–3.67 s 1.17–1.23x slower - herbie.egg 3.44–3.57 s 4.49–4.62 s 1.27–1.33x slower + math.egg 0.941–1.07 s 741–869 ms 0.723–0.885x faster + eggcc-extract.egg 1.44–1.57 s 1.29–1.42 s 0.845–0.959x faster + pointer-analysis-small.egg 1.94–2.07 s 1.94–2.07 s 0.956–1.05x CI includes 1 + hardboiled.egg 2.44–2.57 s 2.69–2.82 s 1.06–1.14x slower + luminal.egg 2.94–3.07 s 3.54–3.67 s 1.17–1.23x slower + herbie.egg 3.44–3.57 s 4.49–4.62 s 1.27–1.33x slower Peak RSS - File Baseline (95% CI) Candidate (95% CI) Ratio (95% CI) Result + File Baseline (95% CI) Candidate (95% CI) Ratio (95% CI) Result ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - math.egg 89.8–101.9 MiB 99.3–111.4 MiB 1.01–1.20x higher RSS - eggcc-extract.egg 108.9–121.0 MiB 118.4–130.5 MiB 1.01–1.16x higher RSS - pointer-analysis-small.egg 127.9–140.1 MiB 137.5–149.6 MiB 1.01–1.14x higher RSS - hardboiled.egg 147.0–159.1 MiB 156.5–168.7 MiB 1.01–1.12x higher RSS - luminal.egg 166.1–178.2 MiB 175.6–187.7 MiB 1.01–1.11x higher RSS - herbie.egg 185.2–197.3 MiB 194.7–206.8 MiB 1.00–1.10x higher RSS + math.egg 89.8–101.9 MiB 99.3–111.4 MiB 1.01–1.20x higher RSS + eggcc-extract.egg 108.9–121.0 MiB 118.4–130.5 MiB 1.01–1.16x higher RSS + pointer-analysis-small.egg 127.9–140.1 MiB 137.5–149.6 MiB 1.01–1.14x higher RSS + hardboiled.egg 147.0–159.1 MiB 156.5–168.7 MiB 1.01–1.12x higher RSS + luminal.egg 166.1–178.2 MiB 175.6–187.7 MiB 1.01–1.11x higher RSS + herbie.egg 185.2–197.3 MiB 194.7–206.8 MiB 1.00–1.10x higher RSS ────────────────────────────────────────────────────── Comparison ────────────────────────────────────────────────────── diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index 2e56a257..35d465eb 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -34,6 +34,10 @@ ) +def _header_positions(lines: list[str], labels: tuple[str, ...]) -> list[tuple[int, ...]]: + return [tuple(line.index(label) for label in labels) for line in lines if all(label in line for label in labels)] + + def test_report_ids_encode_parts_unambiguously() -> None: assert report_id("target", "ab", "c") != report_id("target", "a", "bc") @@ -197,6 +201,26 @@ def test_realistic_six_file_rich_120_snapshot( assert "Other (7 more source rulesets)" in rendered +def test_repeated_rich_table_schemas_share_column_positions(tmp_path: Path) -> None: + report_path, comparison = _six_file_pair_case(tmp_path) + catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") + + for width in (120, 160, 200): + console = Console(record=True, width=width, color_system=None) + console.print(render_rich_report_document(catalog, width)) + lines = console.export_text().splitlines() + + ruleset_positions = _header_positions(lines, ("Driver", "Δ", "Wall share", "Important phase changes")) + assert len(ruleset_positions) == len(comparison.files) + assert len(set(ruleset_positions)) == 1 + + result_positions = _header_positions( + lines, ("File", "Baseline (95% CI)", "Candidate (95% CI)", "Ratio (95% CI)", "Result") + ) + assert len(result_positions) == 2 + assert len(set(result_positions)) == 1 + + def test_detail_level_is_cumulative(tmp_path: Path) -> None: report_path, comparison = _pair_case(tmp_path) expected = { From a78619ec5c70a4e175449e6901722eaa3337d57e Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 09:58:05 -0400 Subject: [PATCH 6/9] Focus benchmark PR on reusable reporting --- encoding-architecture-bridge.md | 577 ------------------------ incremental-unification-pr-roadmap.md | 615 -------------------------- term-encoding-overhead-benchmark.md | 245 ---------- term-encoding-overhead-breakdown.md | 259 ----------- term-encoding-unification.md | 477 -------------------- 5 files changed, 2173 deletions(-) delete mode 100644 encoding-architecture-bridge.md delete mode 100644 incremental-unification-pr-roadmap.md delete mode 100644 term-encoding-overhead-benchmark.md delete mode 100644 term-encoding-overhead-breakdown.md delete mode 100644 term-encoding-unification.md diff --git a/encoding-architecture-bridge.md b/encoding-architecture-bridge.md deleted file mode 100644 index 7eb425de..00000000 --- a/encoding-architecture-bridge.md +++ /dev/null @@ -1,577 +0,0 @@ -# Encodings as semantics, fusion as engineering - -- Status: draft architecture note for paper and maintainer review -- Date: 2026-08-12 -- Audience: egglog maintainers and the encoding-paper authors -- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` -- Implementation permission: none; this document proposes experiments and gates -- Companion: [`term-encoding-unification.md`](term-encoding-unification.md) -- Incremental PR sequence: - [`incremental-unification-pr-roadmap.md`](incremental-unification-pr-roadmap.md) -- Current overhead decomposition: - [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) - -## Executive decision - -Do not choose between the paper's encoding story and a fast, single-path -implementation. Separate two things that the current implementation conflates: - -1. An **encoding is a semantic compiler pass**. It translates a program in an - extended language to a smaller logical language, carries an origin map, and - has a correctness argument. Proofs and slotted e-graphs remain encodings in - this sense and compose in a specified order. -2. A **literal encoded program is only one physical implementation** of that - pass. Production egglog should stage and fuse the encoded operations into - its native tables, union-find, rebuild indexes, and compact evidence arenas. - It should not materialize every administrative relation and then execute the - generated maintenance rules as ordinary user rules. - -The proposed end state is therefore: - -```text -surface program - | - | E_slot (when slotted semantics are requested) - v -typed core program + slot interpretation - | - | E_proof (when proof production is requested) - v -typed encoded program + origin map + optional proof skeletons - | - | normalize, erase unused decorations, and fuse - v -one native physical plan - | - v -one execution engine -``` - -For the paper, the typed encoded program can be printed as ordinary core -egglog and run as a reference semantics. For production, the same artifact is -lowered to a fused plan. The printer is not a separately maintained executor or -a second frontend. - -This permits an honest use of the word *encoding*: the logical translation is -always the source of semantics, while compiler correctness justifies executing -an optimized implementation of it. Compilers do not stop being compilers when -they fuse intermediate allocations away. - -## What should be deleted, and what should survive - -The goal is not to delete the idea of term, proof, or slotted encoding. It is to -delete independent implementations of the same language semantics. - -### Delete from the production path - -- the cloned typechecking `EGraph` and second desugar/typecheck pipeline; -- source-string or source-AST expansion as a prerequisite for execution; -- term-only and proof modes as separately executed egglog programs; -- proof-disabled `Unit` columns threaded through all generated tables; -- generated term/view/`@UF` tables when the native table and union-find already - store the same logical information; -- generated occurrence indexes, path-compression rules, rebuild rules, cleanup - rules, subsumption rules, and schedule injection; -- proof nodes stored as ordinary database rows when a compact side arena or - rule-firing receipt is sufficient; -- `proof_check_program` as a second full command stream; -- backend-selection and lowest-common-denominator abstractions if the project - is in fact removing the alternate backends; -- generic always-on provenance or causal-slicing infrastructure as the - foundation for proofs. - -### Keep as semantic and validation assets - -- the pure equality, proof, and slotted translations; -- the equations and invariants currently documented by the term encoding; -- a deterministic printer for the typed encoded IR, usable by paper examples, - differential tests, and debugging; -- the proof algebra, simplifier, extractor, and independent checker; -- proof skeletons and stable source-rule identities; -- slotted renamings, symmetry/group checks, and an explicit account of fresh - slots; -- origin and interpretation metadata that maps compiled rules and - substitutions back to the source language; -- literal-vs-fused parity tests. - -The reference printer may live in the repository, a paper-artifact crate, or a -test-only feature. That packaging choice is secondary. It must be derived from -the same typed pass result, rather than becoming a second implementation that -can drift. - -## What "one execution path" means - -One path does not mean one feature-free IR node or one enormous interpreter. -It means: - -- one parse, resolution, normalization, and typechecking pipeline; -- one representation of each source rule and function declaration; -- one physical table implementation, union-find, rebuild implementation, and - scheduler; -- optional compile-time decorations for slots and proofs; -- optional evidence storage selected at monomorphization or plan construction, - with a zero-cost `NoEvidence` form; -- no runtime branch that sends a whole program through an independently - maintained semantics. - -The fast native mechanisms do not disappear. Their role changes: they become -the physical lowering of the encoding rather than an independent definition of -surface egglog. - -Likewise, proof production can remain optional without becoming a second path. -A proof-enabled rule is the same normalized rule with a proof plan attached; -it is not a second rule parsed and typechecked in another `EGraph`. - -## A typed encoding contract - -The smallest useful contract is not a generic backend trait. It is a compiler -artifact with four pieces: - -| Piece | Purpose | -| --- | --- | -| Logical program | The actual target-language declarations, rules, actions, and schedules | -| Origin map | Source declaration/rule/action responsible for every generated item | -| Interpretation | How target values, substitutions, and observations map back to the source language | -| Invariants | Facts a physical lowerer may rely on and must preserve, such as canonical views or total slot maps | - -An encoding pass consumes one typed language and produces another typed -language plus this metadata. Its output language must be the next pass's input -language. That makes pass ordering explicit and makes invalid compositions fail -at the compiler boundary rather than in generated source. - -The compiler should preserve administrative operations as typed nodes or typed -annotations long enough for fusion. It must not ask the lowerer to rediscover -them from generated names such as `@UF_Math` or `@AddView`. Name-based -peepholes would retain the entire source generator and introduce a hidden third -path. - -For rules, the conceptual result is: - -```text -normalized rule - + source origin - + optional slot match/action plan - + optional proof skeleton - -> one executable rule plan -``` - -This is a conceptual record, not yet a proposed Rust API. The vertical-slice -experiment should determine which fields are real and which can be derived. - -## Directed composition: slotted first, proofs second - -The project notes already answer the commutativity question: the passes do not -commute; proofs should run after the slotted encoding. That is not a failure of -composition. Composition means that the codomain of the slotted pass is in the -domain of the proof pass: - -```text -E_slot : SlottedEgglog -> CoreEgglog + SlotInterpretation -E_proof : CoreEgglog -> ProofCoreEgglog + ProofInterpretation - -E_both(P) = E_proof(E_slot(P)) -``` - -This order is substantively useful: - -- the slotted pass makes the paper's `beta` and `mp` explicit in rule matching; -- it lowers a union of renamed ids into the base operations that implement it; -- the proof pass then records the actual compiled premises and actions; -- a single rule firing can carry both the slot witness and proof skeleton; -- proof extraction can interpret that firing through both origin maps to name - the original source rule and source-level substitution. - -Arbitrary pass permutation should not be a paper claim. A stronger and more -defensible claim is **typed, directed composition with an interpretation -theorem**. - -### The correctness obligations - -The paper needs to separate four properties that are easy to blur: - -1. **Proof erasure.** Erasing proof decorations from `E_proof(Q)` has the same - observations as running `Q`. -2. **Proof soundness.** Every extracted proof denotes a valid equality in the - semantics of `Q`. -3. **Slotted preservation.** Interpreting the observations of `E_slot(P)` gives - the observations of slotted program `P`. -4. **Physical refinement.** Running the fused physical plan gives the same - observable result as running the literal encoded core program. - -Schematically: - -```text -observe(run_core(E_both(P))) - == observe(run_fused(lower(E_both(P)))) - -decode_slot(erase_proof(observe(run_core(E_both(P))))) - == observe_slotted(P) -``` - -The second equation gives semantic composition. It does **not** by itself give -a pleasant source-level proof. For that, the proof interpretation must map an -encoded rule firing and its renaming witness back to the source slotted rule. -That resugaring/interpretation lemma is a real paper obligation, not metadata -that can be reconstructed after execution. - -## Why the current proof skeleton is the right precedent - -The existing proof design already has a valuable specification/implementation -split: - -- layer 1 describes the proof that would be built as the rule executes; -- layer 2 emits a compact proof skeleton and reconstructs layer 1 later. - -The next engineering step is not to abandon that encoding. It is to stop -storing the skeleton, term identities, and proof nodes as ordinary egglog rows -when the production engine can retain them more compactly. - -In the proposed architecture: - -- layer 1 remains the declarative proof specification used in the paper; -- the typed proof pass derives a skeleton from a normalized rule; -- the fused rule plan stores the static part of that skeleton once; -- a firing records only the dynamic holes that the skeleton needs; -- extraction materializes the existing proof algebra on demand; -- `NoEvidence` erases the skeleton holes and all per-firing writes. - -This is partial evaluation of the proof encoding, not a different proof -semantics. - -## How the literal equality encoding should fuse - -The current term encoding makes several logical objects explicit. A native -lowerer can recognize the typed objects directly and implement them with one -physical structure: - -| Logical encoded object | Fused production representation | -| --- | --- | -| term relation plus canonical view | native constructor/function table, with optional stable `TermId` sidecar | -| explicit per-sort `@UF` | native union-find | -| view collision merge | native congruence/rebuild event | -| occurrence relation/index | native rebuild occurrence index | -| parent/rebuild/cleanup schedule | native commit and rebuild loop | -| `Unit` proof column | erased | -| proof-valued column | compact `CauseId` or receipt sidecar | -| proof-node relations | append-only proof arena materialized on demand | -| generated rule proof | static skeleton plus firing-hole bindings | - -Reaching 5-10% requires essentially all of these fusions. The current -term-only measurements show that frontend cleanup alone is not enough: the -literal relational representation changes row widths, query shapes, write -counts, and maintenance work. - -It is therefore plausible for **encoded semantics with evidence erased** to be -within 5%, because it can lower to nearly the same physical operations as -normal mode. It is not currently plausible for the literal encoded program to -reach that range, nor is there evidence that **always retaining arbitrary proof -evidence** can do so. - -## Slotted-specific implications - -The current slotted rule design is already naturally compiler-shaped: it -computes the paper's `beta` and `mp`, turns each user variable into a leader plus -renaming, and distinguishes fully bound group lookups from genuine -`find-mapping` joins. - -The typed pass should preserve those distinctions. In particular: - -- a known symmetry membership test should stay a lookup, not be expanded into - an enumerating join and then rediscovered by an optimizer; -- extension of `mp` is a real solver operation and should remain explicit; -- union acts on renamed ids, so its origin and renaming witness must survive - into proof interpretation; -- fresh-slot completion must be solved before claiming a complete slotted - encoding; -- the self-edge/group invariant must be stated at phase boundaries, because - the current machinery can expose transient derived facts inside maintenance. - -Some slotted operations may remain relational in the first fused engine. The -architecture does not require every encoding feature to have a native data -structure on day one. It requires one execution plan and an explicit boundary -where a measured hot logical operation can later receive a specialized -physical implementation. - -## Why generic slicing/provenance is not the shared substrate - -The slicing campaign is a useful negative architecture experiment. - -It found that a general recorder plus post-hoc causal reconstruction: - -- added roughly 9,820 lines of provenance recording and 5,300 lines of slicing - including tests, with about +26,182 production lines at PR time; -- still had a witness-free capture floor of 2.213x normal on the decisive Math - experiment; -- required reasoning about row lifetimes, deletes, merge boundaries, - containment, replay identity, and pre-event equality denotation; -- did not become small merely because "any valid support" replaced exact - historical support. - -That does not mean receipts are unusable. It means proof production should not -be implemented as arbitrary execution history followed by a generic graph -query. The proof compiler already knows the rule, its static proof skeleton, -and the exact dynamic holes it needs. Record those holes locally at the rule -and equality-effect boundaries. - -The slicing lesson should become an architectural constraint: - -> No generic recorder, replay engine, or second interpreter may be added to -> support the first proof/slotted vertical slice. - -Slicing can remain out of scope, or later consume an explicitly bounded receipt -interface as a debug feature. It must not define the common runtime substrate. - -## Removing backends changes the paper story - -The older paper pitch used a cross product of expressive features and -performance backends, then claimed that one encoded program was portable over -several backends. If DuckDB, Differential Dataflow, and slicing are being -removed, that claim should be removed rather than simulated by abstractions in -main. - -The replacement story is tighter: - -1. E-graph extensions such as proofs and slotted matching normally cut across - matching, actions, equality, rebuild, extraction, and printing. -2. Expressing each extension as a typed semantic encoding localizes its - definition and makes their order of composition explicit. -3. Literal execution establishes an executable reference semantics. -4. Staging and fusion recover the specialized performance of one production - engine without reintroducing a second language implementation. -5. The implementation is evaluated on semantic parity, composition, - performance, and net production complexity. - -This changes "portability across backends" into **portability of extension -semantics across physical representations**, demonstrated here by a literal -reference execution and one fused execution. If that wording sounds too much -like two backends, omit portability entirely and call the contribution -*composable encodings with semantics-preserving fusion*. - -The paper should not claim that encodings eliminate all extension-specific -engineering. Each encoding still needs a pass, a correctness argument, and -possibly a physical optimization. The claim is that this work is localized and -composes at a declared boundary instead of multiplying through the core. - -## Candidate paper claims and evidence - -| Claim | Required evidence | Current state | -| --- | --- | --- | -| Proofs and slotted semantics are separate encodings | formal definitions plus executable reference translations | proof translation exists; slotted user-rule translation is incomplete | -| The encodings compose | runnable `E_proof(E_slot(P))`, directed composition theorem, source interpretation | not yet demonstrated | -| Fusion preserves the encoding | differential/reference tests plus a physical-refinement argument | absent; proposed work | -| Fusion recovers near-native performance | same-binary literal vs fused vs current-normal benchmarks | absent; current literal term mode is 2.01-2.12x | -| Main becomes simpler | net production LoC, deleted paths, fewer core touchpoints and support gates | absent; must be measured, not asserted | -| Proofs remain independently checkable | existing checker validates source-interpreted composed proofs | checker exists; composed interpretation absent | - -The paper can succeed without 5-10% proof-enabled overhead. A defensible -performance result would report three distinct costs: - -- fused encodings with evidence erased; -- fused proof evidence capture, without extraction; -- proof extraction, simplification, and checking. - -Only the first is the gate for deleting the normal semantic path. Conflating it -with always-on proof capture would make the engineering decision depend on a -much stronger and currently unsupported performance claim. - -## Complexity budget and stop rules - -One engine is not automatically a smaller repository. A typed IR, origin maps, -fusion, and a reference printer can themselves become a large parallel system. -The work should therefore use deletion-backed gates: - -1. **Every production abstraction names the old code it will delete.** A new - pass field or runtime hook is not accepted merely because it may be useful. -2. **No second interpreter.** The reference form is printed from the same typed - artifact and run only by the existing core semantics in tests/artifacts. -3. **No generated-name peepholes.** Fusion operates on typed provenance or - typed operators. -4. **No generic provenance substrate.** The first slice records only holes - demanded by its static proof skeleton. -5. **One vertical slice before broad coverage.** Stop if the slice adds more - production machinery than the old slice it demonstrably replaces. -6. **Keep a running LoC ledger.** Separate production, tests, reference - semantics, and documentation. A smaller core cannot be inferred from a - smaller file count. -7. **Delete as the migration proceeds.** Do not defer all deletion until every - feature is supported; use narrow internal seams so completed families stop - exercising the old path. -8. **Re-measure on every architectural checkpoint.** The slicing campaign - showed that stale cost attribution can steer days of design in the wrong - direction. - -The final deletion gate should require: - -- one frontend/typechecker and one runtime dispatcher; -- one physical equality/rebuild implementation; -- no production execution of printed encoded source; -- no support gate whose only reason is representational inability of the old - generator; -- net production LoC reduction relative to the frozen baseline, or an explicit - maintainer decision that a measured complexity increase is worth the result. - -## Falsifying implementation sequence - -### A0: settle the semantic boundary - -Write down the source and target languages of `E_slot` and `E_proof`, their -observations, pass order, and interpretation maps. Decide whether the composed -proof must name source slotted rules or whether a proof of the compiled core -program is sufficient. - -Stop if the paper team cannot agree on this: the implementation cannot repair -an ambiguous theorem statement. - -### A1: one typed reference artifact - -Change no runtime behavior. Make one tiny constructor/rewrite example produce a -typed encoded artifact from which the current literal core program can be -printed. The artifact must retain source origins without parsing generated -names. - -Gate: printed output behaves exactly like the existing term/proof encoding and -the existing proof checker accepts its proof. - -### A2: fuse one equality/proof vertical slice - -Lower the same artifact directly to the existing native constructor table, -union-find, and rebuild path. Attach one static proof skeleton and record only -its dynamic firing holes under `ProofEvidence`. - -Gates: - -- literal and fused observations match; -- the source-interpreted proof checks; -- `NoEvidence` makes no per-row allocation and adds at most 5% wall time/RSS on - the microcase and a representative existing benchmark; -- the diff includes a named deletion or replacement of the corresponding old - execution branch. - -### A3: measure the proof-capture floor - -Record stable terms and the minimal sound rule/equality receipts, but do not -extract or simplify proofs. This is the optimistic lower bound for proof -availability. - -Do not set 5-10% proof-enabled overhead as a project promise unless this floor -meets it. If it misses, keep evidence optional and proceed with the one-path -design. - -### A4: compose a minimal slotted proof - -Use a program that needs a non-identity renaming and a repeated-variable group -membership check. Run `E_slot` then `E_proof`; compare the literal and fused -forms; extract a proof that names the source rule and carries enough renaming -evidence to check. - -This is the decisive paper slice. Do not begin broad slotted benchmarks until -it works. - -### A5: close known slotted semantic gaps - -Implement and validate fresh-slot completion, settle the phase-boundary group -invariant, and differential-test the translation against the slotted-egraphs -reference implementation. These are correctness gates, not optimization work. - -### A6: migrate semantic families and delete the split - -Move globals/scopes, containers, input, custom merge, delete/subsume, user -indexes, primitives, and extraction one family at a time. Each family must add -parity tests, remove its representation-only rejection, and stop using the old -production path. - -After corpus and proof gates pass, remove the production term/proof execution -mode. Retain only the derived reference printer and paper/test artifacts. - -## Architecture alternatives - -| Alternative | Paper fit | Performance | Complexity outcome | Verdict | -| --- | --- | --- | --- | --- | -| Execute today's literal term encoding universally | strongest superficial dogfooding | current evidence is about 2x term-only and about 3x proofs | deletes native semantics but retains a large generator and maintenance program | reject | -| Typed encodings plus semantics-preserving fusion | keeps encodings and directed composition central | can lower erased mode to current native mechanisms | can delete both independent production paths if deletion gates hold | recommend | -| Native extensible annotation/hook algebra | proofs and slots may share elegant metadata operations | potentially fastest | risks another broad core substrate and weakens the compiler-encoding paper | research alternative, not first slice | -| Paper artifact separate from production main | cleanest immediate main cleanup | production stays fast | paper and engineering may drift; no dogfooding claim | fallback if fusion fails its complexity gate | - -## Open decisions - -1. Must a composed proof name and validate the original slotted rule, or is a - proof over the compiled core rule the paper's theorem? The former is much - more compelling and requires an explicit interpretation lemma. -2. What exactly is `CoreEgglog` for the formalism? It should be small enough to - state semantics, but not chosen as a lowest common denominator for backends - that are being removed. -3. Is the literal printer shipped in main, test-only, or held in the paper - artifact? It must not become a public execution mode by accident. -4. Which stable term identity is genuinely required under `NoEvidence`? Any - always-present identity must earn its measured cost. -5. Can slot and proof plans share a firing substitution without widening every - native binding row? This is a primary performance experiment. -6. What is the accepted production LoC outcome? "Less complexity" needs a - frozen baseline and a measurable deletion target. -7. Is slicing fully out of scope, or a later debug consumer? It should not - influence the first common interface either way. - -## Knowledge-unit map - -| ID | Knowledge unit | Kind | -| --- | --- | --- | -| KU-1 | The current paper direction is proofs plus slotted as composable encodings, with backends and slicing being removed from scope | project decision | -| KU-2 | The intended order is slotted then proofs; the passes do not commute | project decision | -| KU-3 | Literal term-only execution is far outside a 5-10% normal-path gate | measured fact | -| KU-4 | The proof design already separates a declarative layer from emitted skeletons | current design fact | -| KU-5 | The slotted rule translation makes `beta`/`mp` explicit but has unresolved fresh-slot and invariant questions | current design fact | -| KU-6 | General causal recording produced high overhead and a large production diff even after simplification campaigns | measured historical fact | -| KU-7 | Typed fusion can make encoded/no-evidence execution near-native | hypothesis to falsify | -| KU-8 | Composed proof interpretation can recover source-level slotted rules and substitutions | blocked design obligation | -| KU-9 | The architecture will reduce net production complexity | hypothesis to measure | -| KU-10 | Arbitrary proof evidence can always be retained within 5-10% | unsupported stronger hypothesis | - -## Evidence matrix - -| KU | Primary source | Status | Consequence | -| --- | --- | --- | --- | -| KU-1 | project meeting notes, Aug. 5-6; current maintainer direction | Convergent | remove backend portability and slicing from the central architecture | -| KU-2 | project meeting notes lines 102-142 | Convergent | specify typed directed composition, not commutativity | -| KU-3 | `term-encoding-unification.md` fresh same-binary benchmark | Convergent | do not attempt to tune the literal representation to 1.05x | -| KU-4 | `egglog/src/proofs/proof_encoding.md`, proof layers 1 and 2 | Convergent | preserve the semantic layer while changing storage/lowering | -| KU-5 | `slotted-user-rules.md`, fresh-slot gap and open questions | Convergent | composition claims remain gated on slotted correctness work | -| KU-6 | `SLICING-CAMPAIGN-REPORT.md` and its fresh capture-floor experiment | Convergent | prohibit generic recording/replay in the first architecture slice | -| KU-7 | no prototype or benchmark yet | Absent | A2 is a falsifying experiment, not an implementation commitment | -| KU-8 | meeting notes identify resugaring/composition difficulty; no theorem exists | Blocked | A0 must settle the source-level proof contract | -| KU-9 | no fused implementation or deletion diff exists | Absent | use an LoC ledger and named deletion gates | -| KU-10 | current term/proof and slicing capture measurements | Divergent | keep proof evidence optional unless A3 changes the evidence | - -## Source basis - -Highest-authority sources used for this note: - -1. Current maintainer direction in this design discussion: proofs and slotted - remain encodings that compose; alternate backends and slicing are being - removed. -2. `/Users/saul/Downloads/egglog encoding project.md`, especially the Aug. 5-6 - notes on pass ordering, composition, paper claims, and removal of backends - and slicing. -3. [`egglog/src/proofs/proof_encoding.md`](egglog/src/proofs/proof_encoding.md), - the current equality/proof encoding and skeleton design. -4. [`slotted-user-rules.md`](slotted-user-rules.md), the current concrete - slotted user-rule translation and its open semantic gaps. -5. [`term-encoding-unification.md`](term-encoding-unification.md), current-main - benchmark and code-path evidence. -6. `/Users/saul/p/wt/egglog-encoding/pr42-agent-causal-slice-logical-v1/SLICING-CAMPAIGN-REPORT.md`, - the slicing complexity/performance retrospective. - -This note intentionally treats the fusion architecture, its performance, its -net LoC effect, and source-level composed-proof interpretation as proposals. -They are not established by the current sources. - -## Review checklist - -- Does the paper team agree that an encoding is the logical pass, not a mandate - to execute its literal output? -- Is directed `E_proof(E_slot(P))` the intended meaning of composition? -- Is source-level proof interpretation required? -- Are backends and slicing definitively out of the central claims? -- Does A2 delete a named old path before any broad framework is built? -- Are disabled, capture-only, extraction, and checking costs reported - separately? -- Does every complexity claim have an LoC/touchpoint measurement? diff --git a/incremental-unification-pr-roadmap.md b/incremental-unification-pr-roadmap.md deleted file mode 100644 index 1222d3d1..00000000 --- a/incremental-unification-pr-roadmap.md +++ /dev/null @@ -1,615 +0,0 @@ -# Incremental PR roadmap to one engine - -- Status: proposed sequence; no runtime implementation has started -- Date: 2026-08-12 -- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` -- Architecture companions: [`term-encoding-unification.md`](term-encoding-unification.md) - and [`encoding-architecture-bridge.md`](encoding-architecture-bridge.md) -- Current-main performance companion: - [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) - -## Outcome - -There is a credible incremental route to one production engine with optional -proof evidence. It should not begin by replacing the native union-find with -today's ordinary-table `@UF` encoding. It should proceed in this order: - -1. remove the second generated-program frontend; -2. erase proof-only data from the evidence-disabled plan; -3. make the evidence-disabled encoded operations lower to the existing native - tables, relational union-find, and rebuild driver; -4. delete term-only as an independently executed production mode; -5. move proof evidence onto optional sidecars of those same operations; -6. delete the remaining source-generated proof executor one semantic family at - a time. - -The destination has one union-find, not a native UF plus a relational UF: - -```text -logical Equivalence operation - | - v -EquivalenceTable one Table implementation - parents: UnionFind the only canonicalizer - displaced: (child, epoch) relational change stream - reasons?: (left, right, cause, epoch) optional proof sidecar -``` - -`DisplacedTable` already establishes the important precedent: it is relational -at the database boundary and specialized underneath. A proof reason sidecar is -not a second equivalence structure. It explains effective unions made by the -one structure. - -The 5-10% target applies first to **evidence erased**. Full proof capture, -extraction, simplification, and checking must be measured separately. One -engine does not require proofs to be always recorded. - -The current-main decomposition shows that the performance work cannot be one -serial "optimize UF" campaign. Math is maintenance-dominated, Pointer is -frontend-dominated, and Luminal is dominated by transformed user-rule -planning/search plus the generated frontend. The dependency order below still -removes the generated frontend before deleting an executor, but focused -performance PRs should proceed against the discriminator for their own cost -family rather than treating suite aggregate as one mechanism. - -## What the two session investigations change - -### The `single codebase` Claude session - -The local Claude session -`7fd2857d-167e-48c1-9f0c-c3c5f42f97c6` correctly identifies the central -reframe: - -- keep the encoding as the definition of logical semantics; -- treat specialized tables, union-find, and rebuild as a certified physical - implementation of that encoding; -- do not equate "encoded" with "execute every administrative relation and - maintenance rule literally"; -- use a literal form as a differential oracle and paper artifact rather than a - separately maintained production engine. - -It also identifies useful current costs: term relations and `mint-*` have no -proofs-off reader, `Unit` payloads are dead in term-only mode, and the ordinary -table `@UF` duplicates the existing relational `DisplacedTable` facade. - -However, the session's claimed pure-encoding floor of roughly 1.3-1.6x is not a -measurement. Its own adversarial critique calls that number a prior, observes -that Luminal's transformed-search regression had not been explained, and shows -that some proposed pre-frontend performance gates were arithmetically -unreachable while the second frontend remained. This roadmap therefore uses -the session for hypotheses and architecture, not as proof of a floor. - -### The Luminal-overhead Codex session - -The Codex session -`019ff6c4-f93d-70e0-9440-c2f3e97bc4fa` supplies two decisive corrections. - -First, native Luminal's original "outside rulesets" number was partly an -accounting bug. PR -[#61](https://github.com/saulshanabrook/egglog-encoding/pull/61) records direct -experimental scheduler runs. Its six-round comparison left wall time unchanged -while changing the report from 6 to 16 recorded rulesets and reducing the -unattributed share from 97.39% to 55.45%. - -Second, the remaining proof/term frontend cost is real and measured: - -| Luminal typechecking component | Term-only | Proof generation | -| --- | ---: | ---: | -| All generated typechecking | about 187 ms | about 421 ms | -| Standalone actions and lets | about 104 ms | about 283 ms | -| Rules | about 66 ms | about 120 ms | -| Constraint solving | about 52 ms | about 135 ms | -| Primitive overload validation | about 32 ms | about 86 ms | - -Source typechecking itself was only about 34 ms. The generated program's 1,634 -large top-level lets accounted for about 69% of proof-mode typechecking and its -491 rules for another 24%. About 48% of typechecking leaf CPU was allocator or -memory-library work; `PrimitiveWithId::accept` appeared below about 20.5% of -the samples. - -This changes the PR order. The first architectural performance PR should emit -typed generated actions, not optimize the UF. It both removes measured cost -and collapses a duplicated compiler path. Optimizing the general constraint -solver first would tune a path that the typed lowering is intended to delete. - -Proof extraction has a separate small win: actual Luminal spent about 54 ms -gathering 1,634 globals while constructing `ProofStore`, then about 53 ms -gathering them again during `remove_globals`. Reusing that map should remove -one scan. The eggcc fixture instead needs later work on its large proof DAG; -its dominant extraction cost was proof-store conversion, not globals. - -## Sequencing rules - -Every production PR in the critical path should satisfy these rules: - -1. **Name a deletion.** A new type or hook must identify the old parse, - generated relation, maintenance rule, runtime branch, or proof row it - replaces. -2. **Improve a live mode.** Except for the already-open accounting prerequisite - and one paper-composition checkpoint, every PR must either produce a - statistically supported runtime/RSS improvement or delete a production - execution branch with identical performance. -3. **Keep exact mode labels.** Measure `off`, `term`, `proofs`, - `proof-extraction`, and `proof-testing` separately. `proofs` means capture - without automatic extraction or validation. -4. **One variable per benchmark.** Compare the same binary protocol, fixture, - thread count, rounds, and report schema. Preserve anomalous samples. -5. **No generated-name peepholes.** Fusion consumes typed operations or origin - metadata, never names such as `@UF_Math`. -6. **No generic recorder.** Proof capture records only the dynamic holes of a - static proof skeleton. The causal-slicing recorder's optimistic Math floor - was already 2.213x and its campaign added a large amount of code. -7. **Protect the disabled path.** Any proof infrastructure that makes `off` - measurably slower has the wrong layout. A disabled policy should allocate - nothing and should not add dynamic dispatch to row or union hot paths. -8. **Recheck parallelism before deletion.** The current evidence is - single-threaded. The final evidence-erased path must preserve correctness - and performance under the supported multithreaded configuration. - -Each performance PR should carry a small ledger in its description: - -| Item | Required entry | -| --- | --- | -| Baseline | exact base and candidate SHAs | -| Modes | exact treatment names | -| Workloads | focused discriminator plus six-file suite | -| Effect | wall ratio, RSS ratio, and relevant phase/ruleset delta | -| Semantics | proof/corpus parity result | -| Complexity | production lines added, replaced, and deleted | -| Decision | proceed, revise attribution, or stop | - -## Recommended PR queue - -The queue is deliberately front-loaded with changes that improve today's -encoded proof path even if the later fusion design changes. - -| PR | Scope | Current overhead removed | Named deletion or replacement | -| --- | --- | --- | --- | -| P0a | Merge ruleset-accounting PR #61 (complete in `5ead0a0`) | none; makes direct scheduler attribution sound | old outer-only report accumulation | -| P0b | Report rule assembly and disjoint frontend/command stages | none; prevents the remaining residual from being misdiagnosed | wall-minus-rules inference as the primary diagnosis | -| P1 | Reuse proof globals during extraction | one roughly 53 ms Luminal global scan | second `gather_globals` traversal | -| P2 | Emit typed top-level generated actions | largest measured Luminal generated-typecheck bucket | action string generation, reparse, re-desugar, re-typecheck, and generated-global removal for that family | -| P3 | Emit typed generated rules, facts, and schedules | second-largest generated-typecheck bucket and repeated plan setup | corresponding source-string and untyped-command path | -| P4 | Emit typed declarations/merges and remove the second frontend | remaining generated parsing/typechecking and cloned-e-graph bookkeeping | cloned `original_typechecking` `EGraph` and the generated-program frontend loop | -| P5 | Erase proof-only storage under `NoEvidence` | term-row double writes, mint calls, and dead payload width | proofs-off term relations, `mint-*`, and `Unit` proof columns | -| P6 | Make evidence-erased functions/globals identity-lowered | Luminal program/query expansion and plan-construction work | duplicate views, rewritten user queries, and nullary-global view expansion under `NoEvidence` | -| P7 | Route evidence-erased equality through one relational UF | generic-table UF merges and about 43-48 ms Math path compression | per-sort ordinary `@UF` merge programs and `@parent` rules under `NoEvidence` | -| P8 | Route evidence-erased canonicalization through fused rebuild; retire term-only mode | about 382-397 ms Math encoded rebuild plus schedule overhead | generated rebuild/cleanup/subsume schedules and the independently executed term treatment | -| P9 | Add proof reasons to the same `EquivalenceTable` and migrate equality proofs | proof-valued UF rows, eager path proof composition | proof-mode ordinary `@UF`, compression proof rules, and equality proof-node rows | -| P10 | Replace proof term relations with an immutable `TermArena` | append-only per-derivation term rows and mint traffic | constructor/custom term relations and their mint primitives in proof mode | -| P11 | Attach static proof skeletons to the one typed rule plan | duplicated proof-instrumented rules and dynamic proof-node construction | proof rule copy and most proof-node relations | -| P12 | Migrate remaining semantic families and delete the source proof executor | family-specific encoded maintenance and support gates | `ProofInstrumentor` production execution, `proof_check_program`, and representation-only rejections | - -P0 and a later minimal slotted-plus-proof composition test are the only planned -PRs that do not directly reduce overhead or delete an execution branch. - -## PR details and gates - -### P0a: merge accurate direct-ruleset accounting - -PR #61 merged in `5ead0a0cacf847a129294a870de13503f2d7f9c4`. It is an -observability prerequisite, not a performance result. - -The corrected Luminal benchmark is the canary: wall time should remain -statistically unchanged while all 16 native rulesets remain visible and no -ordinary schedule is double-counted. - -### P0b: make the remaining overhead additive - -Add per-ruleset Assembly before Search and split outside-ruleset time into -exclusive leaves under Lowering (Parse, total Typecheck, Other) and Commands -(Install, Actions/input, Other/schedules), plus a derived residual. Total -Typecheck intentionally combines source and generated checking so the -off-versus-encoded delta answers how much checking the encoding added. The -source pass performed in the separate original-typechecker e-graph must still -be charged to that outer total rather than leaking into Lowering / Other. - -The acceptance artifact is the additive six-file table in -[`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md): -the named buckets plus residual must reconstruct process wall time, and the -instrumentation itself must have no detected material tax. Keep this PR scoped -to timing; do not combine it with the shared-globals behavior change from the -larger phase-timer prototype. - -### P1: reuse globals during proof extraction - -Compute the globals environment once for the requested proof and pass or own it -through proof-store construction and global removal. The ownership boundary -should make staleness impossible; this should not become a general cache keyed -by program identity. - -Gate: - -- actual Luminal `proof-extraction` loses one roughly 50 ms scan; -- `proofs` is unchanged, because it performs no extraction; -- eggcc proof-extraction does not regress; -- all proof snapshots and strict proof tests pass. - -This is a useful warm-up but is not on the core unification dependency chain. - -### P2: typed top-level actions first - -Introduce the smallest private generated-program builder needed to construct -resolved variables, calls, expressions, and `ResolvedNCommand::CoreActions`. -Keep declarations on the old path temporarily, but let declaration processing -return the `FuncType` and resolved primitive handles the typed action builder -needs. - -Do not introduce a public general-purpose IR yet. A transitional mixed command -enum is acceptable only if P4 names and deletes it. The invariant owned by the -builder is meaningful: every generated declaration is registered exactly once, -and every emitted call refers to that registered typed object. - -Gate: - -- the 1,634-action Luminal category no longer enters the general constraint - typechecker a second time; -- the proof-generation wall-time CI shows a real improvement; -- term-only also improves; -- emitted behavior and proof propositions are unchanged; -- the PR deletes the old top-level action parse path at its converted call - sites. - -The observed bucket suggests a large win, but no percentage should be promised -until the branch is measured. If the improvement is much smaller than the -removed 104/283 ms typecheck buckets, profile before P3 rather than optimizing -the constraint solver by assumption. - -### P3: typed rules, facts, and schedules - -Construct `ResolvedRule`, resolved facts, and resolved schedules directly. -Preserve rule name, ruleset, evaluation mode, `no_decomp`, -`include_subsumed`, source origin, and the existing proof-head/skeleton layout. -Continue to feed the existing typed-to-core rule machinery so groundedness, -canonicalization, duplicate-variable removal, and rule installation are not -reimplemented. - -Gate: - -- the 491-rule Luminal category no longer runs the second general typechecker; -- generated rule typechecking falls by approximately the measured category, - subject to a fresh profile; -- rule plans and canonical database results match; -- the converted rule/fact/schedule string builders are deleted. - -### P4: typed declarations and deletion of the second frontend - -Finish typed emission for sorts, functions, indexes, merge bodies, and -remaining commands. Separate source type information from target declaration -registration, but do not retain a cloned `EGraph` just to hold the source type -environment. - -At the end, the pipeline should be: - -```text -source parse/desugar/typecheck once - -> typed encoding artifact - -> typed declarations/rules/actions registered and run directly -``` - -Delete the loop in `EGraph::resolve_command` that turns each generated command -back into an unresolved command, desugars it, typechecks it, and removes globals -again. Delete the cloned typechecking-e-graph chain once callers use the typed -artifact. - -Gate: - -- no generated command is reparsed or passed through the general typechecker; -- normal source diagnostics remain source-oriented; -- Luminal and eggcc frontend time falls materially; -- the full support/proof corpus and `make check` pass; -- net production LoC for the frontend split is down, not merely moved. - -### P5: evidence-erased storage diet - -Make proof evidence an explicit compile-time plan policy. Under `NoEvidence`: - -- do not declare or populate immutable term-node relations; -- do not register or call `mint-*` row primitives; -- do not add a `Unit` proof column to views or UF relations; -- let a view/table allocate its default e-class with the same `FreshId` - mechanism used by native constructors. - -Under `ProofEvidence`, preserve current behavior until P9/P10 replace it. This -temporary policy split is acceptable because it is a staged erasure in one -typed artifact, not two source frontends. - -Focused gates: - -- Herbie and Hardboiled wall/RSS, where per-firing build cost should be visible; -- eggcc RSS and total term rows; -- Luminal user-rule search, to test whether row/schema width contributes to its - expansion cost; -- no change to proof generation or checking. - -If build-heavy workloads barely move, revise the overhead ledger before adding -new storage features. - -### P6: evidence-erased identity lowering - -Under `NoEvidence`, one source function should use one physical function table. -The logical term relation and canonical view may remain distinct in the typed -reference artifact, but fusion maps them to the same table. User rule bodies -must therefore keep the original narrow atom shape. Source globals should use -the common `remove_globals` lowering rather than gaining an additional term -relation, FD view, index, and rebuild rule. - -This is the decisive Luminal PR. The current transformed user search is about -475-477 ms versus about 5.3-5.7 ms in `off`, while generated maintenance is only -a few milliseconds. A successful identity lowering should make the same user -rules compile to the same physical query shape. - -Gate: - -- compare ruleset-by-ruleset search, not only whole wall time; -- the evidence-erased user rules are physically shape-equivalent to normal; -- generated rule/index counts for globals disappear; -- if Luminal user search remains more than 2x native, stop and inspect the - remaining plan difference before touching the UF. - -### P7: one relational UF for evidence-erased execution - -Introduce a typed `Equivalence` operation and lower it to the existing -`DisplacedTable`/`UnionFind` implementation. The database continues to -see a table-shaped interface and a displaced-value change stream. The encoder -no longer emits an interpreted ordering merge or a path-compression rule under -`NoEvidence`. - -This PR should prepare, but not prematurely implement, the proof contract: -`union(left, right, optional cause)` and an effective-union event. It must not -add a second parent forest or allocate reasons when evidence is disabled. - -Gate: - -- `@parent` disappears from evidence-erased reports; -- Math improves from removal of its roughly 43-48 ms path-compression ruleset - plus interpreted UF merge overhead; -- min-id leader, push/pop, extraction, and update semantics match; -- off-mode code generation and performance remain unchanged. - -### P8: fused rebuild and retirement of term-only execution - -Lower the typed canonicalization operation to the existing container-first -bulk rebuild/fixpoint driver. Under `NoEvidence`, delete generated occurrence -declarations, row-rewrite rules, cleanup/subsume schedules, and trailing -maintenance schedules. The literal typed artifact can still print those -logical operations for the paper or differential oracle. - -At this point evidence-erased encoding should be physically identical to the -current normal path. Make `term` an alias temporarily only if needed to prove -that identity, then delete it as a production treatment and CLI execution -choice. Retain a test-only literal interpreter configuration if it pays for its -maintenance through the differential oracle. - -Deletion gate: - -- evidence-erased wall and RSS are within 1.05x on every suite file and the - aggregate CI contains 1.0; -- canonical database parity holds for the corpus; -- multithreaded parity and performance pass; -- no production command dispatcher chooses between normal and term-only - programs; -- the PR deletes more production path code than it adds. - -If this gate fails despite supposedly identical physical operations, another -hidden path remains. Do not loosen the target to bless it. - -### P9: proof-capable relational UF, without a second UF - -Extend the same `EquivalenceTable` with a `ProofEvidence` policy. On an -effective union, append a compact reason event naming the two pre-union leaders, -the chosen leader, a `CauseId`, and the epoch. Canonical `find` still consults -the one native UF. Path compression is a storage optimization and does not -create proof nodes. - -Reconstruct equality explanations lazily from the reason forest and lower them -to the existing `ProofStore`/checker format. The first acceptance case is: - -```text -insert a, b, f(a), f(b) -record a = b by a source rule or fiat -derive f(a) = f(b) by congruence -materialize the explanation -accept it with the current independent checker -``` - -Then migrate the current proof `@UF` users and delete proof-valued parent rows, -`Trans`/`Sym` nodes created solely for compression, and the encoded parent -rules. - -Gate: - -- one physical `UnionFind` exists; -- `NoEvidence` remains binary/performance neutral; -- all equality and congruence proof fixtures validate; -- proof-generation Math improves before proceeding to term storage. - -### P10: immutable terms as an arena, not ordinary rows - -Proofs need stable syntactic identity after e-classes move or rows are deleted. -They do not require one append-only database row per derivation attempt. Add an -immutable, hash-consed `TermArena` used only by `ProofEvidence`: - -```text -TermId -> constructor and child TermIds -row/eclass -> witness TermId -CauseId -> source rule, merge, congruence, or fiat receipt -``` - -Move constructor and custom-row proof reconstruction onto this arena, then -delete proof-mode term relations and `mint-*` primitives. - -Gate on proof-generation and proof-extraction separately. Eggcc wall/RSS is the -important discriminator because its large proof DAG, not global scanning, -dominates extraction. - -### P11: one rule plan with an optional proof skeleton - -Preserve the current good idea: the static proof shape is known when a typed -rule is compiled. Attach that skeleton and source origin to the one normalized -rule plan. Under `ProofEvidence`, a successful firing records only the stable -row/term/cause IDs needed to fill its holes. Under `NoEvidence`, the plan has no -holes and emits no receipt. - -This is deliberately not a general causal journal. It does not record arbitrary -history and later search for a proof; the proof compiler specifies exactly -which dynamic values are needed. - -Delete the duplicate proof-instrumented rule, proof-node relations replaced by -the skeleton/receipt pair, and eventually the duplicated command stream used -only by checking. - -Gate: - -- exact proof propositions check even if pretty-printed proof shape changes; -- full eggcc 2mm proof-validating performance is reported distinctly from - capture-only and extraction-only results; -- disabled execution remains unchanged; -- net production LoC trends downward against the frozen encoder baseline. - -### P12: family-by-family migration and final deletion - -Do not put the long tail in one PR. Use separate deletion-backed PRs for: - -1. custom functions and merge bodies; -2. containers and normalization receipts; -3. input, globals, scopes, push/pop, and the Rust API; -4. delete, subsume, user indexes, and extraction; -5. remaining primitive and tuple-output cases. - -Each family PR must remove its old encoder branch and at least one -representation-only unsupported reason. Once the corpus and proof gates pass, -delete production execution through `ProofInstrumentor`, -`proof_check_program`, the cloned proof program, and the proof/term dispatcher. - -The surviving proof assets should be the proof algebra, `ProofStore`, -simplifier, extractor/materializer, independent checker, typed skeletons, -origin maps, and the derived reference printer. - -## Slotted composition checkpoint - -After P11 proves that one typed rule plan can carry a source origin and proof -skeleton, add a narrow paper checkpoint before broad P12 migration: - -1. lower one slotted rule requiring a non-identity renaming; -2. run the proof pass after the slotted pass; -3. execute the fused plan; -4. extract a proof whose interpretation names the source slotted rule and - substitution; -5. compare it with the literal typed encoding and validate it. - -This PR may not improve runtime. Its purpose is to prevent a fast proof-only -architecture from invalidating the paper's actual composition claim. It should -not grow a second interpreter or generic provenance framework. - -## Why this order is preferable to the alternatives - -### Do not start with the UF - -UF work is important for Math, but it cannot explain Luminal's dominant -transformed user search or the generated frontend. Starting there would improve -one component while leaving the two largest cross-workload sources intact. - -### Do not start by optimizing the constraint solver - -The solver and primitive overload validation are hot, but almost all of their -proof/term delta comes from typechecking generated commands a second time. Typed -emission removes that work and reduces code. Solver caching is a fallback only -if source-program typechecking remains material afterward. - -### Do not tune the literal source encoding all the way to 10% - -P5 is a useful measured erasure experiment. P6-P8 intentionally stop treating -the literal tables and maintenance rules as the production representation. -Specialized relational storage is not a betrayal of the encoding; it is the -physical lowering that makes the encoding viable. - -### Do not revive the slicing recorder for proofs - -The slicing campaign showed both the code and capture cost of a broad execution -journal. P9-P11 instead record local, typed causes and only the holes required -by known proof skeletons. - -## Complexity ledger - -Freeze these current baselines before P2: - -- encoder-facing production modules - (`proof_encoding*`, `proof_head`, `proof_fresh`, and - `proof_container_rebuild`): about 6,999 lines; -- the full `egglog/src/proofs` production directory excluding - `proof_tests.rs`: about 11,949 lines; -- `DisplacedTable`: 517 lines, much of which remains as the one physical UF; -- current unsupported-reason count and excluded corpus files; -- command-dispatch branches and benchmark/test treatments. - -Do not score moved native UF/rebuild code as deletion merely because it gets a -more general name. The meaningful complexity wins are: - -- one source typecheck and one target registration path; -- one physical function table per source function when evidence is erased; -- one physical UF and rebuild implementation; -- one normalized rule plan; -- no production execution of generated source; -- fewer support gates and test-matrix axes; -- net production LoC reduction by the final P12 gate. - -## Stop rules - -1. If P2 does not recover a substantial portion of the measured generated - action typecheck bucket, stop and re-profile before P3. -2. If P6 does not collapse Luminal's transformed user search, do not infer that - UF or rebuild work will rescue the 10% target. -3. If P8 cannot make evidence-erased execution physically and measurably - equivalent to normal, do not delete the normal dispatcher. -4. If P9-P11 proof capture misses the accepted proof-enabled gate, keep - evidence optional. This does not invalidate the one-engine design. -5. If any evidence hook taxes `off`, move the policy choice to plan - construction/monomorphization or a separate build; do not accept a permanent - 5-10% tax merely because it is inside the target band. -6. If a new abstraction grows faster than the old family it replaces, stop the - broad migration and retain the typed reference artifact plus current native - lowering. - -## Recommended immediate next action - -Land **P0b, additive phase reporting**, next. Then make **P2, typed top-level -action emission**, the first architectural PR. P1 can land independently as a -small extraction cleanup. - -P2 is the best first commitment because it is simultaneously: - -- supported by a clean profile; -- useful to term and proof modes; -- a reduction in compiler duplication; -- required by any typed encoding/fusion paper story; -- independent of the unresolved proof-storage design; -- a way to establish the builder and measurement discipline needed for every - later PR. - -In parallel, use eggcc as the discriminator for a narrowly scoped -ruleset-assembly experiment: generated `@rebuilding`/`@parent` assembly is -about 253 ms there. Do not assume typed emission alone removes that -per-invocation assembly cost. - -Only after P2-P4 should the project choose exact Rust APIs for the relational -UF sidecar. That keeps the UF design grounded in the typed artifact that will -actually call it, instead of preserving assumptions forced by today's -generated source schema. - -## Source basis - -- Current source at the pinned baseline, especially `EGraph::resolve_command`, - `ProofInstrumentor`, `DisplacedTable`, the bridge rebuild driver, and the - typed/core rule lowering. -- Local Claude session `7fd2857d-167e-48c1-9f0c-c3c5f42f97c6`, titled - `single codebase`, including its research workflow output - `wvkzpajc3.output`. -- Local Codex session `019ff6c4-f93d-70e0-9440-c2f3e97bc4fa`, including the - Luminal phase/typechecking profile and PR #61. -- `/tmp/term-encoding-always-on-bd4752e.jsonl`, the same-binary current-main - benchmark cache described in `term-encoding-unification.md`. -- `/Users/saul/Downloads/egglog encoding project.md`, for the paper's directed - slotted-then-proof composition goal. -- `/Users/saul/p/wt/egglog-encoding/pr42-agent-causal-slice-logical-v1/SLICING-CAMPAIGN-REPORT.md`, - for the generic-recorder complexity and capture-floor stop evidence. diff --git a/term-encoding-overhead-benchmark.md b/term-encoding-overhead-benchmark.md deleted file mode 100644 index 3c258d20..00000000 --- a/term-encoding-overhead-benchmark.md +++ /dev/null @@ -1,245 +0,0 @@ -# Benchmark Report - -## Comparison - -| Role | Target | Git | Treatment | -| --- | --- | --- | --- | -| Baseline | d60202f644249ef565de0ca7c51871fe497440a2 | d60202f64424 | off | -| Candidate | d60202f644249ef565de0ca7c51871fe497440a2 | d60202f64424 | term | - -*10 file(s): math-microbenchmark-rational.egg, eggcc-2mm-pass1.egg, pointer-analysis-initdb.egg (facts: /Users/saul/p/wt/egglog-encoding/term-encoding-always-on/egglog/tests/pointer-analysis-initdb), hardboiled_conv1d_32.egg, luminal-llama.egg, herbie.egg, misaal-hvx-dot-product.egg, churchroad-wide-multiply.egg, dialegg-nmm40.egg, speq-preserved-reference-suite.egg · 6 round(s) per endpoint/file · 120 s timeout per run · Report: /private/tmp/term-encoding-d60202f.jsonl* - -## Summary — d60202f644249ef565de0ca7c51871fe497440a2 term vs d60202f644249ef565de0ca7c51871fe497440a2 off - -| Metric | Scope | File(s) | Ratio (95% CI) | Result | -| --- | --- | --- | ---: | --- | -| Wall time | Suite total | 10 files | 1.62–1.64x | slower | -| Wall time | Lowest-ratio file | churchroad-wide-multiply.egg | 0.708–0.720x | faster | -| Wall time | Highest-ratio file | speq-preserved-reference-suite.egg | 3.55–3.65x | slower | -| Peak RSS | Lowest-ratio file | churchroad-wide-multiply.egg | 1.11–1.13x | higher RSS | -| Peak RSS | Highest-ratio file | pointer-analysis-initdb.egg | 3.78–3.80x | higher RSS | - -*Ratios are candidate / baseline; below 1 is lower and above 1 is higher.* - -## Per-file results - -### Wall time - -| File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | -| --- | ---: | ---: | ---: | --- | -| math-microbenchmark-rational.egg | 413–431 ms | 844–865 ms | 1.98–2.08x | slower | -| eggcc-2mm-pass1.egg | 819–824 ms | 1.19–1.21 s | 1.45–1.48x | slower | -| pointer-analysis-initdb.egg | 58.0–59.8 ms | 135–137 ms | 2.27–2.35x | slower | -| hardboiled_conv1d_32.egg | 112–113 ms | 215–218 ms | 1.91–1.95x | slower | -| luminal-llama.egg | 363–366 ms | 1.24–1.25 s | 3.39–3.43x | slower | -| herbie.egg | 52.7–54.3 ms | 105–108 ms | 1.95–2.03x | slower | -| misaal-hvx-dot-product.egg | 33.9–34.5 ms | 102–104 ms | 2.97–3.05x | slower | -| churchroad-wide-multiply.egg | 1.00–1.01 s | 714–724 ms | 0.708–0.720x | faster | -| dialegg-nmm40.egg | 158–165 ms | 263–270 ms | 1.61–1.69x | slower | -| speq-preserved-reference-suite.egg | 46.6–47.6 ms | 168–171 ms | 3.55–3.65x | slower | - -### Peak RSS - -| File | Baseline (95% CI) | Candidate (95% CI) | Ratio (95% CI) | Result | -| --- | ---: | ---: | ---: | --- | -| math-microbenchmark-rational.egg | 287.4–287.6 MiB | 470.3–470.5 MiB | 1.64–1.64x | higher RSS | -| eggcc-2mm-pass1.egg | 106.6–110.5 MiB | 242.9–248.6 MiB | 2.22–2.31x | higher RSS | -| pointer-analysis-initdb.egg | 40.2–40.3 MiB | 152.2–152.7 MiB | 3.78–3.80x | higher RSS | -| hardboiled_conv1d_32.egg | 41.4–41.8 MiB | 68.0–68.3 MiB | 1.63–1.65x | higher RSS | -| luminal-llama.egg | 117.1–119.9 MiB | 258.4–261.0 MiB | 2.16–2.22x | higher RSS | -| herbie.egg | 20.0–20.1 MiB | 33.7–33.9 MiB | 1.68–1.69x | higher RSS | -| misaal-hvx-dot-product.egg | 31.5–32.1 MiB | 64.9–66.2 MiB | 2.03–2.09x | higher RSS | -| churchroad-wide-multiply.egg | 20.1–20.5 MiB | 22.6–22.8 MiB | 1.11–1.13x | higher RSS | -| dialegg-nmm40.egg | 30.7–31.0 MiB | 98.1–98.3 MiB | 3.17–3.20x | higher RSS | -| speq-preserved-reference-suite.egg | 16.7–17.1 MiB | 35.2–35.4 MiB | 2.07–2.11x | higher RSS | - -## Slowdown decomposition - -| File | Wall Δ | Typecheck | Frontend | Program | Equality | Commands | Residual | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Suite total (10 files) | +1935 ms | +18.1% +350 ms | +17.9% +346 ms | ◆ +29.7% +575 ms | +24.6% +476 ms | +7.12% +138 ms | +2.60% +50.3 ms | -| math-microbenchmark-rational.egg | +433 ms | +0.385% +1.66 ms | +0.417% +1.80 ms | +33.6% +145 ms | ◆ +62.5% +270 ms | +2.64% +11.4 ms | +0.464% +2.01 ms | -| eggcc-2mm-pass1.egg | +380 ms | +17.9% +68.2 ms | +18.6% +70.7 ms | ◆ +29.8% +113 ms | +22.7% +86.3 ms | +9.15% +34.8 ms | +1.80% +6.84 ms | -| pointer-analysis-initdb.egg | +77.3 ms | +2.90% +2.25 ms | +26.9% +20.8 ms | +14.8% +11.4 ms | ◆ +38.7% +29.9 ms | +3.66% +2.83 ms | +13.0% +10.1 ms | -| hardboiled_conv1d_32.egg | +104 ms | +24.2% +25.2 ms | +23.1% +24.1 ms | ◆ +33.2% +34.7 ms | +10.8% +11.3 ms | +5.77% +6.02 ms | +2.83% +2.96 ms | -| luminal-llama.egg | +879 ms | +21.0% +185 ms | +18.1% +159 ms | ◆ +46.4% +408 ms | +5.10% +44.8 ms | +7.40% +65.1 ms | +2.01% +17.7 ms | -| herbie.egg | +53.0 ms | +15.3% +8.11 ms | +17.5% +9.29 ms | ◆ +26.6% +14.1 ms | +22.3% +11.8 ms | +15.4% +8.16 ms | +2.86% +1.52 ms | -| misaal-hvx-dot-product.egg | +68.6 ms | +44.6% +30.6 ms | ◆ +45.7% +31.4 ms | +1.16% +0.794 ms | +2.90% +1.99 ms | +0.709% +0.487 ms | +4.93% +3.39 ms | -| churchroad-wide-multiply.egg | -288 ms | -1.97% +5.67 ms | -2.21% +6.36 ms | ◆ +105% -303 ms | -0.606% +1.74 ms | -0.163% +0.469 ms | -0.496% +1.43 ms | -| dialegg-nmm40.egg | +105 ms | +11.0% +11.5 ms | +9.93% +10.4 ms | ◆ +57.8% +60.5 ms | +16.3% +17.1 ms | +2.75% +2.88 ms | +2.20% +2.30 ms | -| speq-preserved-reference-suite.egg | +122 ms | +9.89% +12.1 ms | +9.27% +11.3 ms | ◆ +73.9% +90.4 ms | +0.691% +0.846 ms | +4.52% +5.52 ms | +1.76% +2.15 ms | - -*The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. Each mechanism cell is its share of that row's wall-time change followed by its signed mean time change. Frontend includes parsing, other lowering, and declaration/install commands. Program rules includes every phase of source-origin rulesets except rebuild. Equality/rebuild combines encoded maintenance rulesets with native rebuild tails. Commands includes actions/input, checks, and other schedules. Shares may be negative or exceed 100% when mechanisms offset. ◆ and bold type mark each row's largest absolute share; contributions below 5% are dimmed and improvements are green in Rich and interactive reports. Signed values carry the same information without styling. Residual is wall time minus every recorded leaf; ! means an endpoint's mean residual is negative.* - -## Ruleset drivers - -*Each panel unfolds the Program and Equality cells from the decomposition. Parent rows exactly match those cells and alone show wall share. Program children contain only source-rule Assembly, Search, Apply, Execution, and Merge; Equality children contain every encoded maintenance ruleset plus one global Native rebuild replaced row. ↳ marks children in every format. Zero children are hidden. Source children are ranked by absolute own-work Δ (top 5 plus an exact per-group Other); every nonzero maintenance child is shown. Important phases include every \|phase Δ\| ≥ max(1 ms, 10% of \|row Δ\|), always include the dominant phase (◆), and appear in Assembly, Search, Apply, Execution, Merge, Rebuild order; … marks omitted nonzero phases.* - -### Ruleset drivers — math-microbenchmark-rational.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +145 ms | +33.6% | ◆ Apply +73.1 ms; Merge +71.1 ms; … | -| ↳ | +145 ms | | ◆ Apply +73.1 ms; Merge +71.1 ms; … | -| Equality/rebuild — net | +270 ms | +62.5% | ◆ Search +280 ms; Apply +59.9 ms; Merge +72.1 ms; Rebuild -142 ms; … | -| ↳ @rebuilding | +371 ms | | ◆ Search +246 ms; Apply +59.4 ms; Merge +65.3 ms; … | -| ↳ @parent | +41.2 ms | | ◆ Search +33.8 ms; Merge +6.83 ms; … | -| ↳ @rebuilding_cleanup | +908 ns | | ◆ Assembly +908 ns | -| ↳ @subsume_ruleset | +223 ns | | ◆ Assembly +223 ns | -| ↳ Native rebuild replaced | -142 ms | | ◆ Rebuild -142 ms | - -*Program + Equality account for +96.1% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* - -### Ruleset drivers — eggcc-2mm-pass1.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +113 ms | +29.8% | Assembly +11.6 ms; ◆ Search +56.8 ms; Apply +25.5 ms; Merge +18.8 ms; … | -| ↳ always-run | +85.6 ms | | Assembly +8.73 ms; ◆ Search +44.5 ms; Apply +19.1 ms; Merge +12.9 ms; … | -| ↳ type-analysis | +8.36 ms | | Search +2.43 ms; ◆ Apply +2.93 ms; Merge +2.47 ms; … | -| ↳ is-resolved | +5.00 ms | | ◆ Search +4.02 ms; … | -| ↳ terms | +3.65 ms | | ◆ Search +1.46 ms; … | -| ↳ terms-helpers | +3.61 ms | | ◆ Search +1.90 ms; … | -| ↳ Other (23 more source rulesets) | +7.11 ms | | Assembly +1.49 ms; ◆ Search +2.46 ms; Apply +1.34 ms; Merge +1.73 ms; … | -| Equality/rebuild — net | +86.3 ms | +22.7% | ◆ Assembly +248 ms; Search +57.6 ms; Execution +11.4 ms; Merge +11.2 ms; Rebuild -247 ms; … | -| ↳ @rebuilding | +282 ms | | ◆ Assembly +203 ms; Search +52.0 ms; … | -| ↳ @parent | +49.3 ms | | ◆ Assembly +42.0 ms; Search +5.56 ms; … | -| ↳ @subsume_ruleset | +2.83 ms | | ◆ Assembly +2.73 ms; … | -| ↳ @rebuilding_cleanup | +67.2 us | | ◆ Assembly +67.2 us | -| ↳ Native rebuild replaced | -247 ms | | ◆ Rebuild -247 ms | - -*Program + Equality account for +52.5% of this file's wall-time change. Source rules shown: 5/28 plus exact Other. Maintenance rules shown: 4/4.* - -### Ruleset drivers — pointer-analysis-initdb.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +11.4 ms | +14.8% | Apply +3.91 ms; ◆ Merge +7.45 ms; … | -| ↳ | +11.4 ms | | Apply +3.91 ms; ◆ Merge +7.45 ms; … | -| Equality/rebuild — net | +29.9 ms | +38.7% | ◆ Search +20.2 ms; Apply +3.89 ms; Merge +9.65 ms; Rebuild -4.66 ms; … | -| ↳ @rebuilding | +19.2 ms | | ◆ Search +12.8 ms; Apply +3.29 ms; Merge +2.69 ms; … | -| ↳ @parent | +15.4 ms | | ◆ Search +7.39 ms; Merge +6.96 ms; … | -| ↳ @rebuilding_cleanup | +1.19 us | | ◆ Assembly +1.19 us | -| ↳ @subsume_ruleset | +785 ns | | ◆ Assembly +785 ns | -| ↳ Native rebuild replaced | -4.66 ms | | ◆ Rebuild -4.66 ms | - -*Program + Equality account for +53.4% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* - -### Ruleset drivers — hardboiled_conv1d_32.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +34.7 ms | +33.2% | ◆ Search +24.2 ms; Apply +4.64 ms; … | -| ↳ | +33.1 ms | | ◆ Search +24.2 ms; Apply +4.02 ms; … | -| ↳ typechecking | +1.06 ms | | ◆ Apply +616 us; … | -| ↳ amx | +531 us | | ◆ Assembly +531 us | -| Equality/rebuild — net | +11.3 ms | +10.8% | Assembly +4.99 ms; ◆ Search +5.96 ms; Apply +2.38 ms; Rebuild -3.75 ms; … | -| ↳ @rebuilding | +12.9 ms | | Assembly +4.07 ms; ◆ Search +4.97 ms; Apply +2.34 ms; … | -| ↳ @parent | +2.12 ms | | ◆ Search +984 us; … | -| ↳ @subsume_ruleset | +22.1 us | | ◆ Assembly +22.1 us | -| ↳ @rebuilding_cleanup | +2.88 us | | ◆ Assembly +2.88 us | -| ↳ Native rebuild replaced | -3.75 ms | | ◆ Rebuild -3.75 ms | - -*Program + Equality account for +44.1% of this file's wall-time change. Source rules shown: 3/3. Maintenance rules shown: 4/4.* - -### Ruleset drivers — luminal-llama.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +408 ms | +46.4% | Assembly +89.0 ms; ◆ Search +314 ms; … | -| ↳ fusion_grow | +170 ms | | ◆ Search +168 ms; … | -| ↳ fusion_pair | +148 ms | | ◆ Search +146 ms; … | -| ↳ direct_kernel | +36.9 ms | | ◆ Search +36.7 ms; … | -| ↳ matmul_backend | +30.5 ms | | ◆ Assembly +76.1 ms; Search -45.9 ms; … | -| ↳ fusion_merge | +14.9 ms | | ◆ Search +14.3 ms; … | -| ↳ Other (11 more source rulesets) | +7.62 ms | | ◆ Assembly +11.8 ms; Search -5.93 ms; … | -| Equality/rebuild — net | +44.8 ms | +5.10% | ◆ Assembly +47.3 ms; Rebuild -5.74 ms; … | -| ↳ @rebuilding | +49.7 ms | | ◆ Assembly +46.5 ms; … | -| ↳ @parent | +627 us | | ◆ Assembly +574 us; … | -| ↳ @subsume_ruleset | +238 us | | ◆ Assembly +147 us; … | -| ↳ @rebuilding_cleanup | +3.58 us | | ◆ Assembly +3.58 us | -| ↳ Native rebuild replaced | -5.74 ms | | ◆ Rebuild -5.74 ms | - -*Program + Equality account for +51.5% of this file's wall-time change. Source rules shown: 5/16 plus exact Other. Maintenance rules shown: 4/4.* - -### Ruleset drivers — herbie.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +14.1 ms | +26.6% | ◆ Assembly +8.33 ms; Apply +3.13 ms; Merge +2.02 ms; … | -| ↳ | +14.1 ms | | ◆ Assembly +8.33 ms; Apply +3.13 ms; Merge +2.02 ms; … | -| Equality/rebuild — net | +11.8 ms | +22.3% | ◆ Search +8.17 ms; Apply +1.92 ms; Merge +2.32 ms; Rebuild -2.01 ms; … | -| ↳ @rebuilding | +11.6 ms | | ◆ Search +6.68 ms; Apply +1.83 ms; Merge +1.82 ms; … | -| ↳ @parent | +2.24 ms | | ◆ Search +1.49 ms; … | -| ↳ @rebuilding_cleanup | +2.48 us | | ◆ Assembly +2.48 us | -| ↳ @subsume_ruleset | +1.60 us | | ◆ Assembly +1.60 us | -| ↳ Native rebuild replaced | -2.01 ms | | ◆ Rebuild -2.01 ms | - -*Program + Equality account for +48.9% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* - -### Ruleset drivers — misaal-hvx-dot-product.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +794 us | +1.16% | ◆ Assembly +675 us; … | -| ↳ | +794 us | | ◆ Assembly +675 us; … | -| Equality/rebuild — net | +1.99 ms | +2.90% | ◆ Assembly +1.57 ms; … | -| ↳ @rebuilding | +2.13 ms | | ◆ Assembly +1.56 ms; … | -| ↳ @parent | +48.8 us | | ◆ Search +25.0 us; … | -| ↳ @rebuilding_cleanup | +223 ns | | ◆ Assembly +223 ns | -| ↳ @subsume_ruleset | +180 ns | | ◆ Assembly +180 ns | -| ↳ Native rebuild replaced | -185 us | | ◆ Rebuild -185 us | - -*Program + Equality account for +4.06% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* - -### Ruleset drivers — churchroad-wide-multiply.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | -303 ms | +105% | ◆ Search -305 ms; … | -| ↳ mapping | -305 ms | | ◆ Search -305 ms; … | -| ↳ transform | +923 us | | ◆ Apply +668 us; … | -| ↳ typing | +741 us | | ◆ Apply +466 us; … | -| ↳ misc | +7.69 us | | ◆ Assembly +7.69 us | -| Equality/rebuild — net | +1.74 ms | -0.606% | ◆ Assembly +1.30 ms; … | -| ↳ @rebuilding | +1.53 ms | | ◆ Assembly +1.09 ms; … | -| ↳ @parent | +213 us | | ◆ Assembly +203 us; … | -| ↳ @rebuilding_cleanup | +1.48 us | | ◆ Assembly +1.48 us | -| ↳ @subsume_ruleset | +1.10 us | | ◆ Assembly +1.10 us | - -*Program + Equality account for +105% of this file's wall-time change. Source rules shown: 4/4. Maintenance rules shown: 4/4.* - -### Ruleset drivers — dialegg-nmm40.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +60.5 ms | +57.8% | ◆ Apply +40.8 ms; Merge +23.5 ms; … | -| ↳ rules | +60.5 ms | | ◆ Apply +40.8 ms; Merge +23.5 ms; … | -| Equality/rebuild — net | +17.1 ms | +16.3% | Assembly +1.93 ms; ◆ Search +15.6 ms; Apply +6.35 ms; Merge +4.13 ms; Rebuild -11.4 ms; … | -| ↳ @rebuilding | +27.5 ms | | ◆ Search +14.9 ms; Apply +6.33 ms; Merge +4.03 ms; … | -| ↳ @parent | +996 us | | ◆ Search +776 us; … | -| ↳ @rebuilding_cleanup | +784 ns | | ◆ Assembly +784 ns | -| ↳ @subsume_ruleset | +319 ns | | ◆ Assembly +319 ns | -| ↳ Native rebuild replaced | -11.4 ms | | ◆ Rebuild -11.4 ms | - -*Program + Equality account for +74.1% of this file's wall-time change. Source rules shown: 1/1. Maintenance rules shown: 4/4.* - -### Ruleset drivers — speq-preserved-reference-suite.egg - -| Driver | Δ | Wall share | Important phase changes | -| --- | ---: | ---: | --- | -| Program rules — own work | +90.4 ms | +73.9% | ◆ Assembly +73.5 ms; Search +9.92 ms; … | -| ↳ parseIR.transform-taco-spmv-csc | +28.5 ms | | ◆ Assembly +20.2 ms; Search +4.92 ms; Execution +3.28 ms; … | -| ↳ parseIR.transform-csparse-spmv-csc-nostruct | +28.1 ms | | ◆ Assembly +19.9 ms; Search +4.88 ms; Execution +3.26 ms; … | -| ↳ parseIR.transform-parboil-hist | +17.0 ms | | ◆ Assembly +16.7 ms; … | -| ↳ parseIR.transform-npb-is-hist | +16.7 ms | | ◆ Assembly +16.6 ms; … | -| ↳ parseIR.expand-parboil-hist | +52.7 us | | ◆ Assembly +29.4 us; … | -| ↳ Other (3 more source rulesets) | +132 us | | ◆ Assembly +81.7 us; … | -| Equality/rebuild — net | +846 us | +0.691% | ◆ Assembly +520 us; … | -| ↳ @rebuilding | +844 us | | ◆ Assembly +477 us; … | -| ↳ @parent | +59.9 us | | ◆ Assembly +41.1 us; … | -| ↳ @rebuilding_cleanup | +1.27 us | | ◆ Assembly +1.27 us | -| ↳ @subsume_ruleset | +1.10 us | | ◆ Assembly +1.10 us | -| ↳ Native rebuild replaced | -61.0 us | | ◆ Rebuild -61.0 us | - -*Program + Equality account for +74.6% of this file's wall-time change. Source rules shown: 5/8 plus exact Other. Maintenance rules shown: 4/4.* diff --git a/term-encoding-overhead-breakdown.md b/term-encoding-overhead-breakdown.md deleted file mode 100644 index 45bf6331..00000000 --- a/term-encoding-overhead-breakdown.md +++ /dev/null @@ -1,259 +0,0 @@ -# Where Current Term-Encoding Time Goes - -## Result - -Term encoding is `1.62–1.64x` slower across the current ten-workload suite, but -there is no single dominant cause. The suite mean adds 1.935 seconds over a -3.083-second `off` baseline: - -| Mechanism | Mean delta | Share of slowdown | -| --- | ---: | ---: | -| Source-rule execution | +575 ms | 29.7% | -| Equality/rebuild, net | +476 ms | 24.6% | -| Typechecking | +350 ms | 18.1% | -| Other frontend/install | +346 ms | 17.9% | -| Commands | +138 ms | 7.12% | -| Residual | +50.3 ms | 2.60% | - -The full generated report is checked in as -[`term-encoding-overhead-benchmark.md`](term-encoding-overhead-benchmark.md). - -The important engineering conclusion is that a native or inline rebuild alone -cannot reach the 5–10% target. Making the entire Equality/rebuild bucket as -cheap as the baseline would reduce the suite point ratio only from `1.63x` to -`1.47x`. Removing all measured non-program overhead would still leave `1.20x` -because transformed source rules remain materially different. Reaching `1.10x` -would require removing about 84% of the current added time, including roughly -317 ms, or 55%, of the net Program bucket even after every positive non-program -delta disappeared. - -## Measurement - -The branch first merged `origin/main` at -`46f69b70d0819b03da110e6e785f91c080d58556`. The measured executable state is -commit `d60202f64424`; the later report-only commit does not change that binary. - -```bash -./bench.py \ - --target @d60202f \ - --compare-target @d60202f \ - --detail rulesets \ - --treatment term \ - --report /tmp/term-encoding-d60202f.jsonl \ - --format markdown -``` - -This collected six fresh `off` and six fresh `term` observations for each of -the ten default workloads: 120/120 runs succeeded. Both endpoints used the -same release binary, workload bytes, fact-directory bytes, timeout, and one -execution thread. Endpoint samples are treated as independent because the -JSONL intentionally stores no round-pair identity. - -The actual wall-time ratios were: - -| Workload | `term / off` (95% CI) | -| --- | ---: | -| Math | 1.98–2.08x | -| eggcc | 1.45–1.48x | -| Pointer analysis | 2.27–2.35x | -| Hardboiled | 1.91–1.95x | -| Luminal | 3.39–3.43x | -| Herbie | 1.95–2.03x | -| Misaal HVX | 2.97–3.05x | -| Churchroad wide multiply | 0.708–0.720x | -| DialEgg NMM40 | 1.61–1.69x | -| SPEQ preserved-reference suite | 3.55–3.65x | - -Churchroad is a useful warning against treating every encoding-induced change -as overhead: its `mapping` Search becomes 305 ms faster, more than offsetting -the added frontend and maintenance work. - -### Timer-tax control - -The extra timers were also measured against a V3-compatible clean control on -the same `5ead0a0` source state. That ten-round, six-workload comparison held -the summary serialization shape fixed while replacing the added timer sites -with zero-valued leaves. The clean suite mean was `1.723830 s`; the instrumented -mean was `1.727558 s`, a `1.00216x` point ratio with a `0.995–1.010x` 95% -interval. This detects no suite-level slowdown and rules out a 5–10% timer tax -for the measured off-mode workload mix. The primary term-versus-off result is -also same-binary, so both of its endpoints pay the retained instrumentation. - -## What “inline rebuilding” can mean - -The measured leaves support two distinct interpretations of “inline”: - -1. **Remove Equality ruleset assembly.** This removes 307 ms of lazy plan - creation and per-invocation executable-ruleset construction, producing an - implied `1.53x` suite ratio. -2. **Make net Equality/rebuild baseline-equivalent.** This removes the entire - 476 ms net responsibility, producing an implied `1.47x` ratio. - -The second number is the optimistic answer to “what if the relational UF and -rebuild were as cheap as native rebuilding?” It is not the cost of one named -ruleset. Encoded maintenance is collective: `@rebuilding`, `@parent`, cleanup, -and subsumption together replace the native rebuild loop. - -Across the suite, generated Equality maintenance adds 894 ms before crediting -the 417 ms of native Rebuild it replaces: - -| Equality phase | Mean delta | -| --- | ---: | -| Assembly | +307 ms | -| Search | +391 ms | -| Apply | +80.5 ms | -| Execution | +14.0 ms | -| Merge | +101 ms | -| Native Rebuild replaced | −417 ms | -| **Net Equality/rebuild** | **+476 ms** | - -So a plan-cache or inline-assembly change attacks a real cost, especially on -eggcc, but it leaves most Equality Search/Apply/Merge work intact. Conversely, -folding the native-rebuild credit into `@rebuilding` would falsely make that -single generated ruleset look cheap and obscure the collective substitution. - -## Derived bounds for the engineering question - -The suite row is a sum of per-file mean deltas, not one process observation. -Its additive cells support simple what-if arithmetic, but that arithmetic is -deliberately not another report table: the combinations are editorial, add no -measurement, and hide that a mechanism can dominate one workload while being -irrelevant to another. - -For the specific always-on design question: - -- matching the baseline's typechecking cost alone implies `1.51x`; matching - other frontend/install alone implies `1.52x`, and matching both implies - `1.40x`; -- eliminating Equality ruleset assembly alone implies `1.53x`, while making - the entire net Equality/rebuild responsibility baseline-equivalent implies - `1.47x`; -- eliminating the net source-rule execution delta implies `1.44x`; and -- even eliminating every positive non-program delta while holding the Program - delta fixed implies about `1.20x`. - -These are point-estimate accounting bounds, not implementation predictions. -They have no confidence intervals and do not model interactions: removing -generated types or identities may also change Program Search, Apply, Merge, or -plan assembly. The per-workload rows below are the primary evidence for -choosing an optimization. - -## Workload narratives - -- **Math:** Equality/rebuild is 62.5% of the slowdown. Generated maintenance - costs 412 ms and replaces 142 ms of native rebuild. This is the clearest - relational-UF target, though changed default-rule Apply and Merge still add - 145 ms. -- **eggcc:** no single mechanism wins. Typecheck plus frontend adds 139 ms, - source rules add 113 ms, net Equality adds 86 ms, and commands add 35 ms. - Equality is assembly-heavy: 248 ms of added assembly is almost exactly - offset by 247 ms of removed native rebuild before Search and execution are - counted. `always-run` carries 86 ms of the Program delta. -- **Pointer analysis:** net Equality is largest at 30 ms, frontend is 21 ms, - and Program is 11 ms. Its 10.1 ms residual is large enough that tiny - sub-mechanism conclusions should remain cautious. -- **Hardboiled:** source rules add 35 ms, while typecheck plus frontend adds - 49 ms. The default ruleset's Search dominates its Program child; check - evaluation is routed symmetrically under Commands rather than appearing as a - term-only default ruleset artifact. -- **Luminal:** Program is 408 ms, 46.4% of the slowdown; Equality is only - 44.8 ms, 5.10%. `fusion_grow` and `fusion_pair` add 170 and 148 ms, almost - entirely Search. Typecheck plus frontend adds another 344 ms. UF work is not - the limiting explanation here. -- **Herbie:** mixed across Program (26.6%), Equality (22.3%), frontend/typecheck - (32.8%), and Commands (15.4%). -- **Misaal HVX:** typecheck plus frontend explains 90.3% of the slowdown. - Program and Equality together explain about 4.1%; a UF optimization would - barely move it. -- **Churchroad:** Program Search improves by 303 ms net, making term encoding - faster overall despite every other top-level mechanism becoming slower. -- **DialEgg:** Program contributes 57.8%, led by Apply and Merge; net Equality - contributes 16.3%. -- **SPEQ:** Program contributes 73.9%, mostly assembly in four transform - rulesets; Equality is below 1%. - -## What this answers—and what it does not - -The additive report now answers: - -- how much slowdown is frontend, source-rule execution, relational - equality/rebuild, commands, or residual; -- whether Equality cost is assembly or execution; -- which source or maintenance rulesets carry Program and Equality changes; and -- the suite sum and the distinct per-workload mechanism mixes. - -It does not identify why a source rule searches or assembles more slowly. For -Luminal, the data localizes the problem to `fusion_grow`/`fusion_pair` Search, -but distinguishing wider tuples, extra identity columns, changed join order, -or greater state churn requires a profiler or a targeted lowering ablation. -Likewise, the derived bounds cannot predict cross-mechanism effects. - -## Measurement design - -Every successful process emits one sorted, open list of exclusive -`path -> nanoseconds` leaves: - -- `typecheck/total`; -- `frontend/{parse,other,install}`; -- `program//`; -- `equality//`; -- `equality/rebuild/` for native rebuild tails; and -- `commands/{actions,check,other}`. - -Rulesets receive an explicit Program or Equality-maintenance role at -declaration time; the report never infers semantics from an `@` prefix. Native -Rebuild and encoded maintenance therefore land under one responsibility before -subtraction. Checks use one command path in both modes. Residual is derived as -wall time minus every recorded leaf and remains the additive self-check. - -The ruleset panel is a literal expansion of the decomposition's two -ruleset-borne columns: - -- `Program rules — own work` excludes source rules' native Rebuild tails; -- `Equality/rebuild — net` contains all maintenance rules and one global - `Native rebuild replaced` child; -- source children are top five by absolute own-work delta plus an exact - `Other`; and -- every nonzero maintenance child is shown. - -Parent rows exactly equal the Program and Equality cells, and children exactly -sum to their parent. No report-time name heuristic or cross-endpoint rebuild -credit is needed. - -## Complexity and minimization - -The retained complexity has five responsibilities: - -| Layer | Required work | Deliberate simplification | -| --- | --- | --- | -| Engine timing | Exclusive process and six-phase ruleset boundaries | Static path slices and one duration map | -| Semantic routing | Program, maintenance, native rebuild, and check ownership | One two-variant role instead of name-prefix inference | -| Transport | Persist exact leaves for later projections | One open segmented-path list; no fixed phase structs or second ruleset schema | -| Analysis | Align endpoint samples and derive residual, mechanisms, and drivers | One generic path-sample ledger; parent/child sums are direct | -| Presentation | Decomposition and per-file drivers | One shared catalog and table renderer for Rich, Markdown, and interactive output | - -The final reduction pass kept the old execution path recognizable and removed -presentation-only alternatives: there is no global phase-rollup model, no -ten-column ruleset table, no duplicated fixed five-bucket wire record, and no -special report-time native-rebuild credit. Counterfactual combinations stay in -the engineering analysis rather than becoming another report model. Further -reduction would either lose the Assembly/Search distinction that separates -plan-cache work from query-shape work or make the signs in the ruleset panel -misleading again. - -## Engineering direction - -The measurements support parallel, falsifiable tracks rather than one “UF -fix”: - -1. eliminate generated parsing, re-typechecking, and installation; -2. cache or fuse Equality assembly, then measure whether Equality execution - can approach native rebuild; -3. restore source-rule physical shapes, especially Luminal Search and SPEQ - assembly; and -4. retain the open ledger while each optimization lands so cross-mechanism - movement remains visible. - -A 5–10% always-on target is possible only if these improvements compose. The -current data rejects both “frontend alone” and “relational UF alone” as -sufficient strategies. diff --git a/term-encoding-unification.md b/term-encoding-unification.md deleted file mode 100644 index cf2ff215..00000000 --- a/term-encoding-unification.md +++ /dev/null @@ -1,477 +0,0 @@ -# One execution path for equality and proofs - -- Status: feasibility brief, not an implementation plan yet -- Date: 2026-08-12 -- Baseline: `origin/main` at `5ead0a0cacf847a129294a870de13503f2d7f9c4` -- Paper/engineering companion: [`encoding-architecture-bridge.md`](encoding-architecture-bridge.md) -- Incremental implementation sequence: - [`incremental-unification-pr-roadmap.md`](incremental-unification-pr-roadmap.md) -- Current overhead decomposition: - [`term-encoding-overhead-breakdown.md`](term-encoding-overhead-breakdown.md) - -## Question - -Can egglog remove the independent normal and term/proof execution paths, make -one path universal, and keep the proof-disabled cost within roughly 5-10% of -today's normal mode? - -There are two materially different versions of that goal: - -1. **One proof-capable execution path, with evidence disabled by default.** - This looks feasible. Its disabled mode can be based on today's native - representation and should be held to a near-zero overhead gate. -2. **Always collect enough evidence to extract arbitrary proofs, within 5-10%.** - This is not supported by current measurements. It needs a separate, - deliberately optimistic lower-bound experiment before becoming a design - constraint. - -## Short answer - -Do not try to make the current generated term program the sole production -representation. On current `main`, term-only mode is **2.01-2.12x** normal wall -time across the six-workload suite and uses **1.59-2.26x** peak RSS. The gap is -not one removable proof feature: term-only already carries `Unit` rather than -real proofs. It comes from the physical representation and compiler pipeline: -extra term/view/UF tables, wider rows, rewritten queries, generated maintenance -rules and schedules, and a second desugar/typecheck pass. - -A single path is still a good complexity goal, but the likely destination is: - -```text -source - -> one parse/desugar/typecheck pipeline - -> typed e-graph operations - -> one equality kernel - - native/fused storage and rebuild - - NoEvidence or ProofEvidence sidecar - -> one rule engine -``` - -The current term encoding is valuable as an executable specification of -equality and proof semantics during migration. Once the typed kernel has parity, -its literal output should stop being a production execution mode. A printer -derived from the same typed encoding artifact can remain for the paper, -differential tests, and debugging without remaining an independently maintained -execution path. - -Calling that destination "term encoding" is reasonable if term encoding means -the semantic decomposition -- stable terms, interning, equivalence, congruence, -and justifications. It should not mean materializing that decomposition as -ordinary user-language tables and rules on the main backend. - -## What is duplicated today - -### Normal path - -```text -parse -> desugar -> typecheck -> remove globals -> typed commands - -> constructor/function tables + native Union actions - -> backend union-find + native rebuild -``` - -### Term/proof path - -```text -parse -> desugar -> typecheck in a cloned EGraph -> proof-normal-form checks - -> remove globals - -> generate new source AST: - term relations + FD views + @UF tables + indexes - rewritten bodies/actions + maintenance rules/schedules - -> desugar again -> typecheck again -> remove generated globals again - -> generic table/rule execution - -> (proof mode) retain the original typed program for checking -``` - -The second path is not a small option around the first. It independently models: - -- constructor creation and interning; -- explicit union and congruence; -- rule-body matching and rule-head construction; -- globals and top-level actions; -- custom functions and merge expressions; -- delete and subsume; -- path compression and rebuilding; -- container canonicalization; -- input loading and extraction; -- proof premises, proof nodes, extraction, simplification, and checking. - -The `egglog/src/proofs` directory currently has about 11,949 lines of -production Rust, excluding `proof_tests.rs`. Roughly 6,999 of those lines are in -the encoding-facing modules (`proof_encoding*`, `proof_head`, `proof_fresh`, and -`proof_container_rebuild`), in addition to an 877-line encoding design document. -Not all of those lines would disappear under a native proof path, but this shows -where the representational duplication lives; the checker, proof algebra, and -proof format are separate assets worth preserving. - -## Current measurements - -All benchmark comparisons used the same release binary at the baseline SHA, -four fresh rounds per endpoint/file, one thread, and a 120-second per-process -timeout. The exact cache is -`/tmp/term-encoding-always-on-bd4752e.jsonl`. - -### Execution and memory - -| Comparison | Suite wall-time ratio (95% CI) | Notable range | Peak-RSS range | -| --- | ---: | ---: | ---: | -| term vs off | 2.01-2.12x | 1.37-1.51x eggcc; 3.28-3.50x Luminal | 1.59-2.26x | -| proofs vs term | 1.48-1.59x | 1.26-1.37x eggcc; 1.70-1.78x Math | 1.37-1.78x | -| proofs vs off | 3.04-3.30x | 1.79-2.00x eggcc; 5.04-5.98x Luminal | 2.19-3.94x | - -`proofs` here means proof data generation for workloads ending in ordinary -`check`s. It does not include automatic proof extraction and verification. - -Per-workload term-only wall time: - -| Workload | Off | Term | Ratio (95% CI) | -| --- | ---: | ---: | ---: | -| Math | 357-395 ms | 817-835 ms | 2.09-2.32x | -| eggcc 2mm pass 1 | 813-885 ms | 1.20-1.25 s | 1.37-1.51x | -| Pointer small | 7.48-8.82 ms | 14.8-17.6 ms | 1.76-2.24x | -| Hardboiled conv1d | 113-121 ms | 218-227 ms | 1.83-1.98x | -| Luminal Llama | 369-391 ms | 1.27-1.30 s | 3.28-3.50x | -| Herbie | 53.7-55.2 ms | 107-111 ms | 1.95-2.05x | - -### Generated-program expansion - -`--mode desugar` exposes the extra compiler and representation work. - -| Workload | Normal output | Term output | Structural change | -| --- | ---: | ---: | --- | -| Math mini | 98 lines / 5,068 bytes | 521 / 29,974 | 24 -> 38 rules; 1 -> 6 schedules; 13 indexes added | -| eggcc 2mm pass 1 | 11,901 / 570,604 | 28,429 / 1,733,236 | 660 -> 1,203 rules; 2 -> 55 schedules; 417 indexes added | -| Luminal Llama | 7,058 / 515,895 | 69,607 / 4,274,991 | 491 -> 2,429 rules; 6 -> 29 schedules; 1,899 indexes added | - -For eggcc, 238 constructors plus 17 functions become 611 generated functions. -For Luminal, 117 constructors plus 1,646 functions become 3,548 functions; -the large static graph's nullary globals are a major contributor. - -Frontend-only `hyperfine` measurements (`--mode desugar`, two warmups, ten -runs, output discarded) were: - -| Workload | Normal mean | Term mean | Ratio | -| --- | ---: | ---: | ---: | -| eggcc 2mm pass 1 | 50.6 +/- 0.4 ms | 207.5 +/- 1.8 ms | 4.10x | -| Luminal Llama | 78.7 +/- 2.8 ms | 505.1 +/- 9.4 ms | 6.42x | - -### Runtime mechanisms - -The phase/ruleset data shows that deleting only the second frontend pass would -not reach the target: - -- On Math, generated `@rebuilding` costs 382-397 ms and `@parent` costs - 43.1-47.6 ms. Native rebuilding is faster even though its 135-179 ms appears - as an explicit cost that term mode reports as zero. -- On Luminal, transformed user-rule search rises from about 5.3-5.7 ms to - 475-477 ms. Generated maintenance is only a few milliseconds there; the - view-based query shape itself is the dominant runtime problem. -- On eggcc, the term frontend adds about 157 ms before execution, while the - full wall-time delta is roughly 0.37 s. Both compiler expansion and runtime - representation matter. - -### Language coverage - -The current support gate has 17 distinct unsupported-reason variants. The -checked-in unsupported snapshot contains 48 files out of 162 non-header, -non-`fail-typecheck` `.egg` corpus files. An always-on path cannot ship until -those are either supported by the common semantics or intentionally removed -from the language. - -Several restrictions are artifacts of the encoding rather than intentional -language semantics: function lookups in actions, tuple outputs, user-written -`begin`, merge action blocks, eq-sort `:no-merge`, user indexes, custom sorts, -and some primitive/container result shapes. A single native proof-capable path -should explain evidence for the underlying operation instead of rejecting the -surface syntax because a generated program cannot express it. - -### Experiment ledger - -| Hypothesis | Distinguishing prediction | Observation | Status | -| --- | --- | --- | --- | -| The second compiler pass explains most term overhead | Recorded runtime phases should be close to normal once outside-of-ruleset time is excluded | Math still spends about 434 ms in generated maintenance; Luminal search rises by about 471 ms | Rejected as a sufficient explanation | -| Generated maintenance is the dominant runtime cost | `@rebuilding` and `@parent` should explain most of every file's delta | True for much of Math, false for Luminal, where transformed user queries dominate | Workload-specific, not sufficient | -| Proof-node construction is the main reason term mode is slow | Term-only, with `Unit` proof columns, should be near normal | Term-only is 2.01-2.12x and 1.59-2.26x RSS | Rejected | -| A fused equality kernel can provide one path near normal cost | A no-evidence seam over native effects should benchmark within 1.05-1.10x | Not yet tested | Active; E1 is the next probe | - -Exact benchmark commands: - -```bash -./bench.py \ - --target . --compare-target . \ - --treatment term --compare-treatment off \ - --rounds 4 --timeout-sec 120 \ - --report /tmp/term-encoding-always-on-bd4752e.jsonl \ - --format markdown --detail rulesets - -./bench.py \ - --target . --compare-target . \ - --treatment proofs --compare-treatment term \ - --rounds 4 --timeout-sec 120 \ - --report /tmp/term-encoding-always-on-bd4752e.jsonl \ - --format markdown --detail phases -``` - -Representative frontend probe: - -```bash -hyperfine --warmup 2 --runs 10 --shell=zsh \ - 'target/release/egglog-experimental --mode desugar benchmarks/luminal-llama.egg >/dev/null 2>&1' \ - 'target/release/egglog-experimental --term-encoding --mode desugar benchmarks/luminal-llama.egg >/dev/null 2>&1' -``` - -## Why the current relational representation misses 5-10% - -The normal backend already implements the same semantic jobs in specialized -data structures: - -- one constructor/function table is both lookup structure and canonical view; -- one native union-find stores equivalence compactly; -- native rebuild uses occurrence information without running user-level rules; -- queries match the original, narrower rows; -- construction does not need a persistent term row, view row, and `Unit` proof - column for every application; -- schedules do not need maintenance spliced after user rulesets; -- source commands are not generated, parsed, and typechecked a second time. - -To bring the current term path near normal, all of those differences would -need to be fused away. At that point the physical implementation would be the -native equality kernel again, preferably behind a cleaner typed interface. - -Backend peepholes that recognize generated names such as `@UF_*` and -`@*View` would demonstrate a performance floor, but they are a poor final -architecture: they preserve the large compiler, couple the backend to generated -syntax, and create a hidden third execution path. - -## Recommended destination - -Use one typed semantic path with two evidence policies, not two programs. - -### 1. A typed equality kernel - -The frontend should lower every language construct once into a small set of -operations with explicit invariants, for example: - -- intern a constructor application and return its e-class; -- read or write a custom function row; -- union two e-classes with a cause; -- commit a batch and rebuild canonical columns; -- apply delete/subsume; -- run a typed rule firing with its substitution. - -The one production engine should implement these with today's fused tables, -union-find, and rebuild indexes. With the alternate backends being removed, -this interface should be chosen for clear semantics and useful compiler -staging, not as a lowest common denominator. `Backend::requires_term_encoding()` -should disappear with the backend split rather than be replaced by another -permanent execution-mode switch. - -### 2. Optional evidence attached to the same effects - -Each equality-producing effect should optionally return/store a compact receipt: - -- top-level or input fact (`Fiat`); -- rule firing and the matched row witnesses; -- explicit union/rewrite; -- constructor interning and congruence collision; -- custom-function merge result; -- rebuild/path-compression edge; -- container rebuild and normalization. - -The disabled policy should allocate nothing and avoid per-row dynamic dispatch. -The enabled policy should write compact IDs into a side arena, not ordinary -egglog relations. Proof expressions should be materialized root-first only when -requested. - -This makes proof availability a property of one runtime, while keeping the hot -representation specialized. - -### 3. Stable terms without a second e-graph - -Proofs need immutable syntactic identity even after rows are canonicalized, -deleted, or subsumed. Preserve that invariant in a compact `TermArena` or row -sidecar: - -```text -TermId -> constructor + child TermIds -row/eclass -> witness TermId -union edge -> CauseId -CauseId -> rule/merge/congruence receipt -``` - -This replaces the persistent term relations and proof-node relations without -losing the information the checker needs. - -### 4. A rule catalog instead of `proof_check_program` - -The checker needs normalized rule definitions, merge definitions, global facts, -and primitive validators. Store those once in an immutable typed `RuleCatalog` -shared by execution and checking. Do not retain a second full command stream -whose shape must stay synchronized with the encoded one. - -### 5. Keep proof semantics, remove encoding mechanics - -Likely keep and adapt: - -- the proof algebra and proof term format; -- `ProofStore`, simplification, and the independent checker; -- deterministic extraction policy; -- immutable term identity and typed primitive validators; -- proof snapshot tests. - -Likely delete or replace: - -- `ProofInstrumentor::add_term_encoding` and command-by-command AST rewriting; -- the cloned `original_typechecking` `EGraph` and second typecheck pass; -- generated term/view/`@UF` tables and `Unit` proof columns; -- generated occurrence-index declarations and maintenance schedules; -- generated path-compression, rebuild, cleanup, and subsume rules; -- proof nodes represented as normal e-graph function rows; -- support rejections caused only by the generated representation; -- `proof_check_program` as a duplicate program; -- the production `--term-encoding` execution mode after migration. - -### Likely implementation seams - -Current source already concentrates several equality effects at useful -boundaries: - -- `EGraph::resolve_command` in `egglog/src/lib.rs` is the frontend split that - should collapse back to one typed pipeline. -- `EGraph::declare_function` chooses constructor `MergeFn::UnionId`; this is - where a common constructor/interner contract can replace proof-specific view - declarations. -- `UnionAction::union` in `egglog/egglog-bridge/src/lib.rs` is the direct native - union write. -- `EGraph::rebuild` in the bridge owns container-first canonicalization and - table rebuild; it needs to report congruence/rebuild causes through the same - optional evidence policy. -- `InPlaceActionBuffer::push_bindings` and its scoped counterpart in - `core-relations/src/free_join/execute.rs` are where a successful rule match - becomes an action batch. - -The last item is probably the hardest design boundary. Native joins currently -need variable values to execute a head; proof reconstruction also needs stable -identities for the body rows that witnessed the match. Widening every binding -with row provenance would damage the disabled hot path. E3 therefore needs to -test a representation that is absent under `NoEvidence` and carries compact row -or receipt identities only under `ProofEvidence`. - -## How incremental desugaring fits - -Term encoding is a useful semantic decomposition of the language, but its -pieces should lower into typed internal operators, not recursively back into -egglog source. - -The migration can therefore be incremental: - -1. Normalize globals, nested expressions, and rule heads once into common typed - IR. -2. Give construction/interning one operator and route both normal and proof - behavior through it. -3. Give union, congruence, custom merge, and rebuild explicit cause-bearing - operators. -4. Move input, containers, delete/subsume, and extraction onto those operators. -5. Add the proof evidence policy and reconstruct the current proof format from - receipts. -6. Retain literal encoded output as a parity oracle while each family moves, - generated from the same typed artifact that feeds the fused lowerer. -7. Delete the old production mode once coverage, proof validity, and - performance gates pass; retain the derived printer only as a test/paper - asset if it remains useful. - -This is a strangler migration around semantic operations, not a flag-day -rewrite and not permanent coexistence of two execution semantics. - -## Options - -| Option | Complexity outcome | Performance outlook | Main risk | -| --- | --- | --- | --- | -| Make today's generated term program universal | Deletes native UF/rebuild, retains the large encoder | Poor without fusing away its defining representation | More compiler/backend pattern coupling; incomplete language | -| Native single path plus optional proof sidecar | Deletes the source encoder and support split | Disabled mode can be close to current normal; enabled cost unknown | Capturing sound merge/rebuild/rule causes in the native engine | -| Typed encoding IR plus one fused physical lowering | One language semantics, one engine, and a derived reference printer | Fused lowering can retain current native speed | Designing a stable semantic/fusion boundary without building another framework | -| Keep both paths but isolate/shared utilities | Smaller near-term refactor | No forced regression | Does not remove semantic duplication or support drift | - -The recommendation is the second and third options together: a common typed -encoding IR, one fused native kernel, an optional proof-evidence policy, and a -reference printer derived from that same IR. The companion architecture note -explains how slotted then proof encoding can compose at this boundary. - -## Falsifying experiment ladder - -Large production edits should wait until these floors are measured in order. - -### E0: frozen reference matrix - -Keep the current off/term/proofs measurements and add exact output parity for -the six benchmark files. This is the immutable comparison set. - -### E1: `NoEvidence` seam - -Route native construction, union, merge, and rebuild through the proposed -evidence interface, with a zero-sized disabled implementation. Record nothing. - -Gate: - -- no semantic or snapshot delta; -- <=1.05x suite wall time and <=1.10x on every file; -- <=1.05x peak RSS; -- no per-row allocation and no dynamic dispatch in the hot loop. - -If this fails, the interface boundary is wrong before proof design begins. - -### E2: immutable-term floor - -Record only the stable `TermId`/witness arena needed by any native proof design. -Do not record union causes or build proof nodes. - -This isolates the irreducible cost of keeping syntactic identity. If it already -exceeds 1.10x, reuse existing row IDs more aggressively or abandon an -always-recording 5-10% target. - -### E3: receipt-only floor - -Record the smallest sound cause for native rule firings, unions, congruence, -merge, and rebuild. Do not extract, simplify, or verify a proof. - -This is the decisive optimistic lower bound for "proofs always available at -5-10%." If it misses the gate, selector or extractor work cannot rescue the -capture cost. - -### E4: one end-to-end witness - -On a tiny fixture containing construction, a rewrite, congruence, and a custom -merge, reconstruct the existing proof format from receipts and validate it with -the independent checker. Compare exact propositions, not necessarily exact -pretty-print shape. - -### E5: semantic expansion - -Add containers, globals/scopes, input, delete/subsume, action lookups, tuple -outputs, and user indexes one family at a time. Every accepted family must flip -its current unsupported canary while preserving the existing normal corpus. - -### E6: deletion gate - -Delete the old source encoding only after: - -- every non-failing corpus file uses the common path; -- all explicit proof fixtures validate; -- the six-file disabled-evidence suite stays within the agreed wall/RSS gate; -- proof-enabled overhead is reported separately from disabled overhead; -- the printed reference encoding, if retained, is not callable as a separate - production execution mode. - -## Decision - -The current source-to-source term encoding cannot plausibly be tuned from -2.01-2.12x to 1.05-1.10x by deleting a few proof features. Reaching that band -requires removing the generated physical representation: duplicate tables, -query expansion, maintenance rules, schedule injection, and the second compiler -pass. - -One execution path is nevertheless plausible and likely the best way to reduce -repo complexity. Build it from the native fast path, make equality/provenance -explicit in a typed encoding IR, and make evidence an optional sidecar. Use the -literal term encoding as the semantic oracle during migration, then delete its -production path while retaining a derived reference printer if the paper and -tests still need it. From 0dd081cc4d30b2ca2912f48771360e8008306496 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 11:38:09 -0400 Subject: [PATCH 7/9] Simplify benchmark timing storage --- README.md | 125 +--- benchmarking/reports/analysis.py | 465 ++++++--------- benchmarking/reports/presentation.py | 267 +++++---- benchmarking/reports/render.py | 27 +- benchmarking/reports/store.py | 34 +- egg-math-benchmark/src/main.rs | 67 +-- .../tests/scheduler_reporting.rs | 15 +- egglog/egglog-reports/src/lib.rs | 548 ++++++++++-------- egglog/src/ast/mod.rs | 8 +- egglog/src/lib.rs | 225 +++---- egglog/src/phase_timers.rs | 74 --- egglog/src/scheduler.rs | 23 +- egglog/tests/integration_test.rs | 2 +- egglog/tests/timing_summary_cli.rs | 134 +++-- tests/report_fixtures.py | 58 +- tests/test_collection.py | 41 +- tests/test_report_analysis.py | 305 ++++------ tests/test_report_rendering.py | 33 +- tests/test_report_store.py | 23 +- 19 files changed, 1108 insertions(+), 1366 deletions(-) delete mode 100644 egglog/src/phase_timers.rs diff --git a/README.md b/README.md index c018efe3..b229d9b8 100644 --- a/README.md +++ b/README.md @@ -325,100 +325,17 @@ Every successful benchmark observation records timing from the same measured process. Timing collection is always enabled; requesting a detailed report does not rerun a diagnostic process or change the cache key. -The JSONL stores one sorted list of exclusive timing leaves. Each leaf has a -segmented `path` and a raw nanosecond total; parent totals are never stored. -Segmented paths keep a ruleset name such as `rules/λ` or one containing `/` -unambiguous. - -Ruleset leaves have one of these shapes: - -- `program//` for source-origin ruleset work; -- `equality//` for encoded equality-maintenance work; -- `equality/rebuild/` for native Rebuild tails from source rulesets as - well as rebuild tails from maintenance rulesets. - -Rulesets receive an explicit semantic role when declared. Generated equality -maintenance is therefore not inferred from an `@` name prefix. Moving both -native and encoded implementations under `equality` makes their net cost an -ordinary candidate-minus-baseline difference. - -The recorded ruleset phases are: - -- Ruleset assembly: lazy cached-plan creation and per-invocation executable - ruleset construction. -- Search: matching and join execution. -- Apply: executing rule-head instructions and staging updates. -- Execution: measured pre-merge work that cannot be accurately classified as - Search or Apply. -- Merge: resolving and installing staged updates. -- Rebuild: rebuilding indexes and e-graph state. - -The engine measures one contiguous pre-merge interval, including per-run setup, -and records the remainder after Search and Apply as Execution. Assembly done -inside native rebuild remains part of Rebuild rather than being counted twice. -All six leaves are retained for every invoked ruleset, including zero values. - -The same list contains these process leaves outside ordinary ruleset execution: - -- `typecheck/total`: total source and generated typechecking, including source - checking performed by the encoded mode's cloned checker; -- `frontend/parse`, `frontend/other`, and `frontend/install`: parsing, other - lowering work, and post-resolution execution of declarations; -- `commands/actions`, `commands/check`, and `commands/other`: actions/input, - complete check evaluation, and other commands or schedule driving. - -Check queries are transient backend rules in both encoded and unencoded modes. -Their backend execution and surrounding compilation/validation overhead are -charged together to `commands/check`; they do not appear as program rulesets. -One known command boundary remains: if a top-level action such as `(union ...)` -causes `flush_updates` to rebuild, that rebuild stays in `commands/actions`. -Top-level actions are timed as commands and their transient backend report is -not inserted into the named-ruleset ledger, so this work does not appear under -Equality. - -Residual is derived per observation as external wall time minus every recorded -leaf. It includes process setup, reporting, teardown, and any still- -uninstrumented work. - -At `--detail phases`, the additive slowdown-decomposition table has a suite row -and one row per file. The suite row is the sum of each selected file's -candidate-minus-baseline endpoint mean; it is not a single process observation. -Its rendered headers are `Wall Δ`, `Typecheck`, `Frontend`, `Program`, -`Equality`, `Commands`, and `Residual`. Every mechanism cell displays its share -of the row's wall-time change first, then candidate-minus-baseline milliseconds. -`◆` marks the largest absolute mechanism share in each row; Rich and -interactive reports also bold that cell, dim contributions below 5%, and color -improvements green. Expected overhead is neutral rather than red; warning and -error colors are reserved for suspect measurements. Percentages may be -negative or exceed 100% when mechanisms offset. `!` on a Residual cell means at -least one endpoint's mean recorded total exceeded its wall time. - -At `--detail rulesets`, one compact driver table appears per file. Its -`Program rules — own work` and `Equality/rebuild — net` parent rows exactly -match the corresponding cells in the decomposition, show their wall share, -and report what fraction of the file's wall-time change they jointly account -for. Child rows use a `↳` prefix in Rich, Markdown, and interactive reports. - -Program children contain only source-rule Assembly, Search, Apply, Execution, -and Merge. They never inherit the native Rebuild tail that happened to follow -their invocation. At most five changed source rulesets are ranked by absolute -own-work difference; `Other (N more source rulesets)` is the exact additive -sum of the omitted source children. Equality children contain every changed -encoded-maintenance ruleset and, when nonzero, one global -`Native rebuild replaced` row. Thus the children beneath each parent add -exactly to that parent without a cross-mechanism reconciliation convention. -Zero children are omitted. - -Only parent rows show wall share. Every row retains a compact phase summary. -That summary includes every phase whose absolute change is at least -`max(1 ms, 10% of |row change|)`, always includes the dominant phase marked -with `◆`, and uses fixed Assembly, Search, Apply, Execution, Merge, Rebuild -order. `…` means smaller nonzero phase changes were omitted from display, not -from accounting. - -Rich gives every repeated table schema one content-derived column layout, so -wall/RSS results align with each other and all per-file driver panels retain -the same scan positions. Markdown remains width-independent. +The versioned timing summary stores seven fixed process counters, one typed row +per named ruleset with its Program or Equality role and five exclusive own-work +phases, and one global native-Rebuild counter. Parent mechanisms, shares, and +Residual are derived rather than stored; the same canonical per-file breakdown +feeds both the decomposition and ruleset-driver views, so their parent totals +match by construction. + +Checks are charged to the command counters in both modes. One known boundary is +that a rebuild triggered by a top-level action such as `(union ...)` remains in +Commands/Actions. The captions printed next to `--detail phases` and +`--detail rulesets` are the source of truth for grouping and display rules. Benchmarks run single-threaded. This keeps Search and Apply attribution additive for egglog's interleaved executor. @@ -554,10 +471,9 @@ Each observation contains target and workload provenance, exact cache coordinates, status, wall time, peak RSS, and failure details. A top-level report schema version covers both the persisted shape and measurement semantics, so methodology changes cannot silently reuse stale -measurements. Successful observations also contain the version-3 timing -summary: one open list of exclusive `{path: [segment, ...], ns: value}` leaves. -Adding detail below an existing responsibility prefix does not require another -parallel record shape. Changes to timing coverage or meaning still require a +measurements. Successful observations also contain the version-4 timing +summary: fixed process counters, a typed list of named ruleset timings, and one +global native-Rebuild counter. Changes to timing coverage or meaning require a schema-version change so stale measurements cannot be reused silently. Timed-out rows have null wall time, peak RSS, and timing summary. Failed rows @@ -569,18 +485,17 @@ This tool is the only supported reader and writer. The codec rejects old report and timing-summary schema versions and requires successful rows to contain timing data. It trusts the tool's typed writer rather than repeating the `TypedDict` as runtime field-by-field validation. A schema change intentionally -invalidates existing caches: move or remove an incompatible report and recompute -it. Analysis invariants use ordinary exceptions rather than `assert`, so -optimized Python does not silently accept a persisted Residual leaf or an -unknown top-level timing responsibility. +invalidates existing caches: move or remove an incompatible report and +recompute it. ### Report-analysis ownership `ComparisonSpec` owns the exact endpoints, files, rounds, and timeout; `store.py` owns physical row order and cache selection. `analysis.py` computes -immutable summary, file, mechanism-decomposition, and ruleset rows, while `presentation.py` maps -them to the renderer-neutral catalog. Rich, Markdown, and the interactive page -serialize that catalog without recomputing report facts. +immutable summary and file comparisons plus one canonical timing breakdown per +file. `presentation.py` projects that breakdown into mechanism and ruleset +tables; Rich, Markdown, and the interactive page serialize the catalog without +recomputing report facts. ## Statistics diff --git a/benchmarking/reports/analysis.py b/benchmarking/reports/analysis.py index cd4811de..2ca545ee 100644 --- a/benchmarking/reports/analysis.py +++ b/benchmarking/reports/analysis.py @@ -1,8 +1,8 @@ """Compute renderer-neutral statistics for one benchmark endpoint pair. This module selects observations, estimates means and confidence intervals, -computes Fieller ratios, exhaustively attributes wall time, and ranks changed -rulesets. Persistence lives in :mod:`benchmarking.reports.store`; all labels, +computes Fieller ratios, exhaustively attributes wall time, and partitions +ruleset work. Persistence lives in :mod:`benchmarking.reports.store`; all labels, units, and presentation policy live in :mod:`benchmarking.reports.presentation`. """ @@ -10,6 +10,7 @@ import math import statistics +from collections.abc import Iterable from typing import Literal, NamedTuple, cast from scipy import stats @@ -20,16 +21,13 @@ MetricName = Literal["wall_sec", "max_rss_bytes"] ResultClass = Literal["higher", "invalid", "lower", "point_only", "unclear"] SummaryKind = Literal["suite", "lowest_file", "highest_file"] -MechanismName = Literal["typecheck", "frontend", "program", "equality", "commands", "residual"] RulesetPhaseName = Literal["assembly", "search", "apply", "execution", "merge", "rebuild"] RulesetMechanism = Literal["program", "equality"] -RulesetRowKind = Literal["aggregate", "ruleset", "native_rebuild", "other"] type _MetricKey = tuple[int, int, MetricName] type _ObservationKey = tuple[int, int] -type _TimingPath = tuple[str, ...] _METRICS: tuple[MetricName, ...] = ("wall_sec", "max_rss_bytes") -_RULESET_PHASES: tuple[RulesetPhaseName, ...] = ( +RULESET_PHASES: tuple[RulesetPhaseName, ...] = ( "assembly", "search", "apply", @@ -37,16 +35,6 @@ "merge", "rebuild", ) -_RULESET_MECHANISMS: tuple[RulesetMechanism, ...] = ("program", "equality") -_MECHANISMS: tuple[MechanismName, ...] = ( - "typecheck", - "frontend", - "program", - "equality", - "commands", - "residual", -) -RULESET_CONTRIBUTOR_LIMIT = 5 class Estimate(NamedTuple): @@ -75,12 +63,9 @@ class PhaseValues(NamedTuple): merge: float rebuild: float - -class RulesetDelta(NamedTuple): - """One exact total delta and its six timing-component deltas.""" - - total: float - phases: PhaseValues + @property + def total(self) -> float: + return math.fsum(self) class SummaryView(NamedTuple): @@ -102,43 +87,52 @@ class FileComparisonView(NamedTuple): ratio: RatioEstimate -class SlowdownCell(NamedTuple): - """One mechanism's delta and share of the observed wall slowdown.""" +class RulesetChange(NamedTuple): + """One named ruleset's own-work phase changes.""" + + name: str + phases: PhaseValues - delta_ns: float | None - slowdown_share: float | None +class RulesetGroup(NamedTuple): + """One mechanism's named rulesets and optional global rebuild change.""" -class SlowdownValues(NamedTuple): - """The six additive mechanism cells displayed for one row.""" + rulesets: tuple[RulesetChange, ...] + native_rebuild_delta_ns: float = 0.0 - typecheck: SlowdownCell - frontend: SlowdownCell - program: SlowdownCell - equality: SlowdownCell - commands: SlowdownCell - residual: SlowdownCell + @property + def phases(self) -> PhaseValues: + values = [math.fsum(ruleset.phases[index] for ruleset in self.rulesets) for index in range(len(RULESET_PHASES))] + values[-1] += self.native_rebuild_delta_ns + return PhaseValues(*values) -class SlowdownDecompositionView(NamedTuple): - """One per-file or suite-wide additive slowdown decomposition.""" +class FileTimingBreakdown(NamedTuple): + """One canonical additive timing partition consumed by both timing views.""" file_order: int | None wall_delta_ns: float | None - mechanisms: SlowdownValues + typecheck_delta_ns: float + frontend_delta_ns: float + program: RulesetGroup + equality: RulesetGroup + commands_delta_ns: float + residual_delta_ns: float residual_warning: bool issue: str | None - -class RulesetContributorView(NamedTuple): - """One mechanism parent, named child, native rebuild, or exact remainder.""" - - file_order: int - kind: RulesetRowKind - mechanism: RulesetMechanism - name: str - ruleset_count: int - delta: RulesetDelta + @property + def mechanism_deltas(self) -> tuple[float | None, ...]: + if self.issue is not None: + return (None,) * 6 + return ( + self.typecheck_delta_ns, + self.frontend_delta_ns, + self.program.phases.total, + self.equality.phases.total, + self.commands_delta_ns, + self.residual_delta_ns, + ) class PairReportViewData(NamedTuple): @@ -146,8 +140,7 @@ class PairReportViewData(NamedTuple): summary: tuple[SummaryView, ...] files: tuple[FileComparisonView, ...] - decomposition: tuple[SlowdownDecompositionView, ...] - rulesets: tuple[RulesetContributorView, ...] + timing: tuple[FileTimingBreakdown, ...] class _MetricEstimate(NamedTuple): @@ -157,12 +150,15 @@ class _MetricEstimate(NamedTuple): issue: str | None -class _TimingAggregate(NamedTuple): - """Aligned samples for the open timing paths in one endpoint/file selection.""" +class _TimingMean(NamedTuple): + """One endpoint/file's direct means from the typed timing record.""" - observation_count: int - paths: dict[_TimingPath, list[float]] - residuals: list[float] + typecheck_ns: float + frontend_ns: float + commands_ns: float + rulesets: dict[tuple[RulesetMechanism, str], PhaseValues] + native_rebuild_ns: float + residual_ns: float | None def analyze_pair( @@ -180,16 +176,12 @@ def analyze_pair( summary = _summary_rows(comparison, estimates, file_rows, t_critical) if detail == "summary": - return PairReportViewData(summary, (), (), ()) + return PairReportViewData(summary, (), ()) if detail == "files": - return PairReportViewData(summary, file_rows, (), ()) + return PairReportViewData(summary, file_rows, ()) - timing = _timing_aggregates(observations) - decomposition = _slowdown_decomposition(comparison, timing, issues, estimates) - if detail == "phases": - return PairReportViewData(summary, file_rows, decomposition, ()) - rulesets = _ruleset_contributors(comparison, timing, issues) - return PairReportViewData(summary, file_rows, decomposition, rulesets) + timing = _timing_breakdowns(comparison, observations, issues, estimates) + return PairReportViewData(summary, file_rows, timing) def _selected_observations( @@ -362,267 +354,154 @@ def _summary_rows( return tuple(rows) -def _slowdown_decomposition( +def _timing_breakdowns( comparison: ComparisonSpec, - timing: dict[_ObservationKey, _TimingAggregate], + observations: dict[_ObservationKey, tuple[IndexedRecord, ...]], issues: dict[_ObservationKey, str | None], metric_estimates: dict[_MetricKey, _MetricEstimate], -) -> tuple[SlowdownDecompositionView, ...]: - points: dict[tuple[int, int, MechanismName], float | None] = {} - for (endpoint_order, file_order), aggregate in timing.items(): - for mechanism in _MECHANISMS: - issue = issues[(endpoint_order, file_order)] - if mechanism == "residual" and issue is None: - issue = metric_estimates[(endpoint_order, file_order, "wall_sec")].issue - paths = [path for path in aggregate.paths if path[0] == mechanism] - values = aggregate.residuals if mechanism == "residual" else _sum_path_samples(aggregate, paths) - points[(endpoint_order, file_order, mechanism)] = ( - statistics.fmean(values) if issue is None and values else None - ) - - result: list[SlowdownDecompositionView] = [] +) -> tuple[FileTimingBreakdown, ...]: + means = _timing_means(observations, metric_estimates) + files: list[FileTimingBreakdown] = [] for file_order in range(len(comparison.files)): - baseline_wall = metric_estimates[(0, file_order, "wall_sec")].estimate.point - candidate_wall = metric_estimates[(1, file_order, "wall_sec")].estimate.point + baseline = means[(0, file_order)] + candidate = means[(1, file_order)] + baseline_wall = metric_estimates[(0, file_order, "wall_sec")] + candidate_wall = metric_estimates[(1, file_order, "wall_sec")] + issue = issues[(0, file_order)] or issues[(1, file_order)] or baseline_wall.issue or candidate_wall.issue wall_delta_ns = ( None - if baseline_wall is None or candidate_wall is None - else (candidate_wall - baseline_wall) * 1_000_000_000.0 + if issue is not None or baseline_wall.estimate.point is None or candidate_wall.estimate.point is None + else (candidate_wall.estimate.point - baseline_wall.estimate.point) * 1_000_000_000.0 ) - cells: list[SlowdownCell] = [] - for mechanism in _MECHANISMS: - baseline_point = points[(0, file_order, mechanism)] - candidate_point = points[(1, file_order, mechanism)] - delta = None if baseline_point is None or candidate_point is None else candidate_point - baseline_point - cells.append(SlowdownCell(delta, _share(delta, wall_delta_ns))) - baseline_residual = points[(0, file_order, "residual")] - candidate_residual = points[(1, file_order, "residual")] - issue = ( - issues[(0, file_order)] - or issues[(1, file_order)] - or metric_estimates[(0, file_order, "wall_sec")].issue - or metric_estimates[(1, file_order, "wall_sec")].issue + residual_delta_ns = ( + 0.0 + if baseline.residual_ns is None or candidate.residual_ns is None + else candidate.residual_ns - baseline.residual_ns ) - result.append( - SlowdownDecompositionView( + files.append( + FileTimingBreakdown( file_order, wall_delta_ns, - SlowdownValues(*cells), - (baseline_residual is not None and baseline_residual < 0) - or (candidate_residual is not None and candidate_residual < 0), + candidate.typecheck_ns - baseline.typecheck_ns, + candidate.frontend_ns - baseline.frontend_ns, + _ruleset_group_delta(baseline, candidate, "program"), + _ruleset_group_delta(baseline, candidate, "equality"), + candidate.commands_ns - baseline.commands_ns, + residual_delta_ns, + (baseline.residual_ns is not None and baseline.residual_ns < 0) + or (candidate.residual_ns is not None and candidate.residual_ns < 0), issue, ) ) - suite_issue = next((row.issue for row in result if row.issue is not None), None) - if suite_issue is None: - suite_wall_delta = sum(cast(float, row.wall_delta_ns) for row in result) - suite_cells = [] - for mechanism_index in range(len(_MECHANISMS)): - delta = sum(cast(float, row.mechanisms[mechanism_index].delta_ns) for row in result) - suite_cells.append(SlowdownCell(delta, _share(delta, suite_wall_delta))) - else: - suite_wall_delta = None - suite_cells = [SlowdownCell(None, None) for _ in _MECHANISMS] - suite = SlowdownDecompositionView( + suite_issue = next((row.issue for row in files if row.issue is not None), None) + suite = FileTimingBreakdown( None, - suite_wall_delta, - SlowdownValues(*suite_cells), - any(row.residual_warning for row in result), + None if suite_issue is not None else math.fsum(cast(float, row.wall_delta_ns) for row in files), + math.fsum(row.typecheck_delta_ns for row in files), + math.fsum(row.frontend_delta_ns for row in files), + _sum_ruleset_groups(row.program for row in files), + _sum_ruleset_groups(row.equality for row in files), + math.fsum(row.commands_delta_ns for row in files), + math.fsum(row.residual_delta_ns for row in files), + any(row.residual_warning for row in files), suite_issue, ) - return (suite, *result) + return (suite, *files) -def _timing_aggregates( +def _timing_means( observations: dict[_ObservationKey, tuple[IndexedRecord, ...]], -) -> dict[_ObservationKey, _TimingAggregate]: - result: dict[_ObservationKey, _TimingAggregate] = {} + metric_estimates: dict[_MetricKey, _MetricEstimate], +) -> dict[_ObservationKey, _TimingMean]: + result: dict[_ObservationKey, _TimingMean] = {} for key, rows in observations.items(): - aggregate = _TimingAggregate(len(rows), {}, []) - for observation_index, row in enumerate(rows): + typecheck = 0.0 + frontend = 0.0 + commands = 0.0 + native_rebuild = 0.0 + rulesets: dict[tuple[RulesetMechanism, str], list[float]] = {} + for row in rows: record = row.record if record["status"] != "success": - for samples in aggregate.paths.values(): - samples.append(0.0) continue summary = record["timing_summary"] if summary is None: raise ValueError("successful benchmark record is missing its timing summary") - observation: dict[_TimingPath, float] = {} - recorded = 0.0 - for leaf in summary["timings"]: - path = tuple(leaf["path"]) - if not path: - raise ValueError("timing path must not be empty") - if path[0] == "residual": - raise ValueError("residual is derived rather than recorded") - if path[0] not in _MECHANISMS[:-1]: - raise ValueError(f"unknown timing responsibility {path[0]!r}") - duration = float(leaf["ns"]) - observation[path] = observation.get(path, 0.0) + duration - recorded += duration - for path, samples in aggregate.paths.items(): - samples.append(observation.pop(path, 0.0)) - for path, duration in observation.items(): - aggregate.paths[path] = [0.0] * observation_index + [duration] - wall_sec = record["wall_sec"] - if wall_sec is not None: - aggregate.residuals.append(wall_sec * 1_000_000_000.0 - recorded) - result[key] = aggregate - return result - - -def _sum_path_samples(aggregate: _TimingAggregate, paths: list[_TimingPath]) -> list[float]: - """Add selected exclusive leaves observation by observation.""" - - return [math.fsum(aggregate.paths[path][index] for path in paths) for index in range(aggregate.observation_count)] - - -def _ruleset_contributors( - comparison: ComparisonSpec, - timing: dict[_ObservationKey, _TimingAggregate], - issues: dict[_ObservationKey, str | None], -) -> tuple[RulesetContributorView, ...]: - """Unfold Program and Equality into truthful per-file child partitions.""" - - result: list[RulesetContributorView] = [] - for file_order in range(len(comparison.files)): - if issues[(0, file_order)] is not None or issues[(1, file_order)] is not None: - continue - source_names = sorted( - { - path[2] - for endpoint_order in (0, 1) - for path in timing[(endpoint_order, file_order)].paths - if len(path) == 3 and path[0] == "program" - } + typecheck += summary["typecheck_ns"] + frontend += summary["frontend_parse_ns"] + summary["frontend_other_ns"] + summary["frontend_install_ns"] + commands += summary["commands_actions_ns"] + summary["commands_check_ns"] + summary["commands_other_ns"] + native_rebuild += summary["native_rebuild_ns"] + for timing in summary["rulesets"]: + phase_sums = rulesets.setdefault((timing["role"], timing["name"]), [0.0] * 6) + phase_sums[0] += timing["assembly_ns"] + phase_sums[1] += timing["search_ns"] + phase_sums[2] += timing["apply_ns"] + phase_sums[3] += timing["execution_ns"] + phase_sums[4] += timing["merge_ns"] + + denominator = len(rows) or 1 + ruleset_means = { + key: PhaseValues(*(value / denominator for value in values)) for key, values in rulesets.items() + } + typecheck /= denominator + frontend /= denominator + commands /= denominator + native_rebuild /= denominator + recorded = ( + typecheck + + frontend + + commands + + native_rebuild + + math.fsum(phases.total for phases in ruleset_means.values()) ) - maintenance_names = sorted( - { - path[2] - for endpoint_order in (0, 1) - for path in timing[(endpoint_order, file_order)].paths - if len(path) == 3 and path[0] == "equality" and path[1] != "rebuild" - } + wall = metric_estimates[(key[0], key[1], "wall_sec")].estimate.point + result[key] = _TimingMean( + typecheck, + frontend, + commands, + ruleset_means, + native_rebuild, + None if wall is None else wall * 1_000_000_000.0 - recorded, ) + return result - names_by_mechanism = {"program": source_names, "equality": maintenance_names} - children: dict[RulesetMechanism, list[RulesetContributorView]] = {"program": [], "equality": []} - for mechanism in _RULESET_MECHANISMS: - for name in names_by_mechanism[mechanism]: - delta = _ruleset_phase_deltas(timing, file_order, name, mechanism) - if any(delta.phases): - children[mechanism].append(RulesetContributorView(file_order, "ruleset", mechanism, name, 1, delta)) - children[mechanism].sort(key=lambda row: (-abs(row.delta.total), row.name)) - - source_rebuild_deltas: list[float] = [ - rebuild_delta - for name in source_names - if ( - rebuild_delta := _ruleset_phase_delta( - timing, - file_order, - name, - "equality", - "rebuild", - ) - ) - != 0 - ] - if source_rebuild_deltas: - rebuild_delta = math.fsum(source_rebuild_deltas) - children["equality"].append( - RulesetContributorView( - file_order, - "native_rebuild", - "equality", - "", - 0, - RulesetDelta( - rebuild_delta, - PhaseValues(0.0, 0.0, 0.0, 0.0, 0.0, rebuild_delta), - ), - ) - ) - for mechanism in _RULESET_MECHANISMS: - group = children[mechanism] - result.append( - RulesetContributorView( - file_order, - "aggregate", - mechanism, - "", - sum(row.ruleset_count for row in group), - _sum_ruleset_deltas(group), - ) - ) - if mechanism == "program" and len(group) > RULESET_CONTRIBUTOR_LIMIT: - omitted = group[RULESET_CONTRIBUTOR_LIMIT:] - result.extend(group[:RULESET_CONTRIBUTOR_LIMIT]) - result.append( - RulesetContributorView( - file_order, - "other", - mechanism, - "", - len(omitted), - _sum_ruleset_deltas(omitted), - ) - ) - else: - result.extend(group) - return tuple(result) - - -def _ruleset_phase_delta( - timing: dict[_ObservationKey, _TimingAggregate], - file_order: int, - name: str, - responsibility: RulesetMechanism, - phase: RulesetPhaseName, -) -> float: - """Subtract one named responsibility/phase mean across the endpoints.""" - - def mean(endpoint_order: int) -> float: - aggregate = timing[(endpoint_order, file_order)] - paths: list[_TimingPath] = [ - path - for path in aggregate.paths - if len(path) == 3 and path[0] == responsibility and path[1] == phase and path[2] == name - ] - if not paths: - return 0.0 - return statistics.fmean(_sum_path_samples(aggregate, paths)) - - return mean(1) - mean(0) - - -def _ruleset_phase_deltas( - timing: dict[_ObservationKey, _TimingAggregate], - file_order: int, - name: str, - responsibility: RulesetMechanism, -) -> RulesetDelta: - """Return own-work Program phases or complete Equality-maintenance phases.""" - - phases = PhaseValues( - *( - 0.0 - if responsibility == "program" and phase == "rebuild" - else _ruleset_phase_delta(timing, file_order, name, responsibility, phase) - for phase in _RULESET_PHASES +def _ruleset_group_delta( + baseline: _TimingMean, + candidate: _TimingMean, + mechanism: RulesetMechanism, +) -> RulesetGroup: + names = sorted({name for role, name in baseline.rulesets.keys() | candidate.rulesets.keys() if role == mechanism}) + zero = PhaseValues(0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + rulesets = [] + for name in names: + baseline_phases = baseline.rulesets.get((mechanism, name), zero) + candidate_phases = candidate.rulesets.get((mechanism, name), zero) + phases = PhaseValues( + *(candidate_phases[index] - baseline_phases[index] for index in range(len(RULESET_PHASES))) ) + if any(phases): + rulesets.append(RulesetChange(name, phases)) + rebuild = candidate.native_rebuild_ns - baseline.native_rebuild_ns if mechanism == "equality" else 0.0 + return RulesetGroup(tuple(rulesets), rebuild) + + +def _sum_ruleset_groups(groups: Iterable[RulesetGroup]) -> RulesetGroup: + """Combine file-level ruleset groups while preserving named phase totals.""" + + phase_sums: dict[str, list[float]] = {} + native_rebuild = 0.0 + for group in groups: + native_rebuild += group.native_rebuild_delta_ns + for ruleset in group.rulesets: + values = phase_sums.setdefault(ruleset.name, [0.0] * len(RULESET_PHASES)) + for index, value in enumerate(ruleset.phases): + values[index] += value + rulesets = tuple( + RulesetChange(name, PhaseValues(*values)) for name, values in sorted(phase_sums.items()) if any(values) ) - return RulesetDelta(math.fsum(phases), phases) - - -def _sum_ruleset_deltas(rows: list[RulesetContributorView]) -> RulesetDelta: - """Sum a ruleset partition without losing phase-level additivity.""" - - phases = PhaseValues(*(math.fsum(row.delta.phases[index] for row in rows) for index in range(len(_RULESET_PHASES)))) - return RulesetDelta(math.fsum(row.delta.total for row in rows), phases) + return RulesetGroup(rulesets, native_rebuild) def _sample_estimate( @@ -642,9 +521,3 @@ def _sample_estimate( ci_low = mean - half_width ci_high = mean + half_width return _MetricEstimate(len(values), Estimate(mean, ci_low, ci_high), var_mean, issue) - - -def _share(numerator: float | None, denominator: float | None, *, scale: float = 1.0) -> float | None: - if numerator is None or denominator is None or denominator == 0: - return None - return numerator / (denominator * scale) diff --git a/benchmarking/reports/presentation.py b/benchmarking/reports/presentation.py index 8a6360f0..e36a7fd1 100644 --- a/benchmarking/reports/presentation.py +++ b/benchmarking/reports/presentation.py @@ -15,17 +15,15 @@ from ..engines import TREATMENT_SPECS from ..models import BenchmarkEndpoint, ComparisonSpec, DetailLevel, FileSpec from .analysis import ( - RULESET_CONTRIBUTOR_LIMIT, + RULESET_PHASES, Estimate, FileComparisonView, + FileTimingBreakdown, MetricName, - PairReportViewData, + PhaseValues, RatioEstimate, ResultClass, - RulesetContributorView, - RulesetDelta, - SlowdownCell, - SlowdownDecompositionView, + RulesetGroup, SummaryView, analyze_pair, ) @@ -47,12 +45,20 @@ NULL = "—" DEFAULT_RULESET = "" +RULESET_CONTRIBUTOR_LIMIT = 5 DETAIL_ORDER: dict[DetailLevel, int] = { "summary": 0, "files": 1, "phases": 2, "rulesets": 3, } +RESULT_TONES: dict[ResultClass, CellTone] = { + "higher": "default", + "invalid": "error", + "lower": "positive", + "point_only": "muted", + "unclear": "muted", +} RATIO_DIRECTION = "Ratios are candidate / baseline; below 1 is lower and above 1 is higher." DECOMPOSITION_CAPTION = ( "The Suite total row sums each selected file's candidate − baseline mean; file rows are per-file mean deltas. " @@ -93,9 +99,9 @@ def build_report_catalog( if _includes(detail, "files"): sections.append(_files_section(views.files, comparison, file_labels)) if _includes(detail, "phases"): - sections.append(_phases_section(views.decomposition, comparison, file_labels)) + sections.append(_phases_section(views.timing, comparison, file_labels)) if _includes(detail, "rulesets"): - sections.append(_rulesets_section(views, comparison, file_labels)) + sections.append(_rulesets_section(views.timing, comparison, file_labels)) return ReportCatalog(tuple(sections)) @@ -319,7 +325,7 @@ def _files_section( def _phases_section( - rows: Sequence[SlowdownDecompositionView], + rows: Sequence[FileTimingBreakdown], comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: @@ -333,17 +339,23 @@ def _phases_section( file = comparison.files[row.file_order] row_id = report_id("row", "phases", file.sha256, file.fact_directory_sha256) label = file_labels[file] - comparable = [index for index, cell in enumerate(row.mechanisms) if cell.slowdown_share is not None] - leader = max(comparable, key=lambda index: abs(row.mechanisms[index].slowdown_share or 0.0), default=None) - if leader is not None and row.mechanisms[leader].slowdown_share == 0.0: + deltas = row.mechanism_deltas + wall_delta = row.wall_delta_ns + shares = tuple( + None if delta is None or wall_delta is None or wall_delta == 0 else delta / wall_delta for delta in deltas + ) + comparable = [index for index, share in enumerate(shares) if share is not None] + leader = max(comparable, key=lambda index: abs(shares[index] or 0.0), default=None) + if leader is not None and shares[leader] == 0.0: leader = None mechanism_cells = tuple( _slowdown_cell( - cell, + delta, + shares[index], leader=index == leader, - warning=row.residual_warning and index == len(row.mechanisms) - 1, + warning=row.residual_warning and index == len(deltas) - 1, ) - for index, cell in enumerate(row.mechanisms) + for index, delta in enumerate(deltas) ) report_rows.append( _row( @@ -378,17 +390,23 @@ def _phases_section( return ReportSection("phases", "Slowdown decomposition", (table,)) -def _slowdown_cell(cell: SlowdownCell, *, leader: bool, warning: bool) -> ReportCell: - duration = _format_delta_ms(cell.delta_ns) - share = _format_percent(cell.slowdown_share, signed=True) +def _slowdown_cell( + delta_ns: float | None, + slowdown_share: float | None, + *, + leader: bool, + warning: bool, +) -> ReportCell: + duration = _format_delta_ms(delta_ns) + share = _format_percent(slowdown_share, signed=True) marker = "◆ " if leader else "" - display = NULL if cell.delta_ns is None else f"{marker}{share} {duration}" + display = NULL if delta_ns is None else f"{marker}{share} {duration}" if warning: display = f"!{display}" return text_cell( - cell.slowdown_share, + slowdown_share, display, - tone=_delta_tone(cell.delta_ns, share=cell.slowdown_share, emphasis=leader, warning=warning), + tone=_delta_tone(delta_ns, share=slowdown_share, emphasis=leader, warning=warning), ) @@ -415,28 +433,19 @@ def _delta_tone( def _rulesets_section( - views: PairReportViewData, + timing: Sequence[FileTimingBreakdown], comparison: ComparisonSpec, file_labels: dict[FileSpec, str], ) -> ReportSection: - by_file: dict[int, list[RulesetContributorView]] = {} - for row in views.rulesets: - by_file.setdefault(row.file_order, []).append(row) - file_issues = { - row.file_order: row.ratio.issue - for row in views.files - if row.metric == "wall_sec" and row.ratio.issue is not None - } - wall_deltas = {row.file_order: row.wall_delta_ns for row in views.decomposition if row.file_order is not None} - blocks: list[ReportBlock] = [] - if views.rulesets: - blocks.append(ReportMessage(report_id("message", "rulesets", "guide"), None, RULESET_CAPTION, tone="muted")) + by_file = {row.file_order: row for row in timing if row.file_order is not None} + blocks: list[ReportBlock] = [ + ReportMessage(report_id("message", "rulesets", "guide"), None, RULESET_CAPTION, tone="muted") + ] for file_order, file in enumerate(comparison.files): title = f"Ruleset drivers — {file_labels[file]}" - rulesets = by_file.get(file_order, []) - if not rulesets: - issue = file_issues.get(file_order) - status = f"Status: {issue}" if issue is not None else "No nonzero ruleset timing differences." + breakdown = by_file.get(file_order) + if breakdown is None or breakdown.issue is not None: + status = f"Status: {breakdown.issue}" if breakdown is not None else "Timing unavailable." blocks.append( ReportMessage( report_id("message", "rulesets", file.sha256, file.fact_directory_sha256), @@ -445,14 +454,13 @@ def _rulesets_section( ) ) continue - wall_delta = wall_deltas.get(file_order) - parents = {row.mechanism: row for row in rulesets if row.kind == "aggregate"} - program_parent = parents["program"] - equality_parent = parents["equality"] + program = sorted(breakdown.program.rulesets, key=lambda row: (-abs(row.phases.total), row.name)) + maintenance = sorted(breakdown.equality.rulesets, key=lambda row: (-abs(row.phases.total), row.name)) + wall_delta = breakdown.wall_delta_ns coverage = ( None if wall_delta is None or wall_delta == 0 - else (program_parent.delta.total + equality_parent.delta.total) / wall_delta + else (breakdown.program.phases.total + breakdown.equality.phases.total) / wall_delta ) coverage_text = ( "Program + Equality coverage is unavailable because wall time did not change." @@ -462,24 +470,95 @@ def _rulesets_section( "of this file's wall-time change." ) ) - source_count = program_parent.ruleset_count + source_count = len(program) source_shown = min(source_count, RULESET_CONTRIBUTOR_LIMIT) source_text = f"Source rules shown: {source_shown}/{source_count}" source_text += " plus exact Other." if source_count > source_shown else "." - maintenance_count = equality_parent.ruleset_count + maintenance_count = len(maintenance) maintenance_text = ( "Maintenance rules shown: none." if maintenance_count == 0 else f"Maintenance rules shown: {maintenance_count}/{maintenance_count}." ) caption = f"{coverage_text} {source_text} {maintenance_text}" + report_rows = [ + _ruleset_report_row( + file, + "aggregate", + "program", + "", + source_count, + breakdown.program.phases, + breakdown.wall_delta_ns, + ) + ] + report_rows.extend( + _ruleset_report_row( + file, + "ruleset", + "program", + ruleset.name, + 1, + ruleset.phases, + breakdown.wall_delta_ns, + ) + for ruleset in program[:RULESET_CONTRIBUTOR_LIMIT] + ) + if len(program) > RULESET_CONTRIBUTOR_LIMIT: + omitted = tuple(program[RULESET_CONTRIBUTOR_LIMIT:]) + report_rows.append( + _ruleset_report_row( + file, + "other", + "program", + "", + len(omitted), + RulesetGroup(omitted).phases, + breakdown.wall_delta_ns, + ) + ) + report_rows.append( + _ruleset_report_row( + file, + "aggregate", + "equality", + "", + maintenance_count, + breakdown.equality.phases, + breakdown.wall_delta_ns, + ) + ) + report_rows.extend( + _ruleset_report_row( + file, + "ruleset", + "equality", + ruleset.name, + 1, + ruleset.phases, + breakdown.wall_delta_ns, + ) + for ruleset in maintenance + ) + if breakdown.equality.native_rebuild_delta_ns != 0: + report_rows.append( + _ruleset_report_row( + file, + "native_rebuild", + "equality", + "", + 0, + PhaseValues(0, 0, 0, 0, 0, breakdown.equality.native_rebuild_delta_ns), + breakdown.wall_delta_ns, + ) + ) blocks.append( _table( report_id("table", "rulesets", file.sha256, file.fact_directory_sha256), title, ("driver", "delta", "share", "important_phases"), ("Driver", "Δ", "Wall share", "Important phase changes"), - tuple(_ruleset_report_row(file, row, wall_delta) for row in rulesets), + tuple(report_rows), caption=caption, alignments=("left", "right", "right", "left"), ) @@ -487,54 +566,55 @@ def _rulesets_section( return ReportSection("rulesets", "Ruleset drivers", tuple(blocks)) -def _ruleset_report_row(file: FileSpec, row: RulesetContributorView, wall_delta: float | None) -> ReportRow: - parent = row.kind == "aggregate" - share = None if not parent or wall_delta is None or wall_delta == 0 else row.delta.total / wall_delta - tone = _delta_tone(row.delta.total, share=share) +def _ruleset_report_row( + file: FileSpec, + kind: str, + mechanism: str, + name: str, + ruleset_count: int, + phases: PhaseValues, + wall_delta: float | None, +) -> ReportRow: + parent = kind == "aggregate" + share = None if not parent or wall_delta is None or wall_delta == 0 else phases.total / wall_delta + tone = _delta_tone(phases.total, share=share) + if kind == "aggregate": + label = "Program rules — own work" if mechanism == "program" else "Equality/rebuild — net" + elif kind == "native_rebuild": + label = "↳ Native rebuild replaced" + elif kind == "other": + label = f"↳ Other ({ruleset_count} more source rulesets)" + else: + label = f"↳ {DEFAULT_RULESET if name == '' else name}" return _row( report_id( "row", "rulesets", file.sha256, file.fact_directory_sha256, - row.kind, - row.mechanism, - row.name, + kind, + mechanism, + name, ), - text_cell( - row.name, - _ruleset_contributor_label(row), - tone="emphasis" if parent else "default", - ), - text_cell(row.delta.total, format_duration(row.delta.total, signed=True), tone=tone), + text_cell(name, label, tone="emphasis" if parent else "default"), + text_cell(phases.total, format_duration(phases.total, signed=True), tone=tone), text_cell(share, _format_percent(share, signed=True) if parent else "", tone=tone), - text_cell(_important_phase_changes(row.delta), tone=tone), + text_cell(_important_phase_changes(phases), tone=tone), ) -def _ruleset_contributor_label(row: RulesetContributorView) -> str: - if row.kind == "aggregate": - return "Program rules — own work" if row.mechanism == "program" else "Equality/rebuild — net" - if row.kind == "native_rebuild": - return "↳ Native rebuild replaced" - if row.kind == "other": - return f"↳ Other ({row.ruleset_count} more source rulesets)" - name = DEFAULT_RULESET if row.name == "" else row.name - return f"↳ {name}" - - -def _important_phase_changes(delta: RulesetDelta) -> str: - labels = ("Assembly", "Search", "Apply", "Execution", "Merge", "Rebuild") - changed = [index for index, value in enumerate(delta.phases) if value != 0] +def _important_phase_changes(phases: PhaseValues) -> str: + changed = [index for index, value in enumerate(phases) if value != 0] if not changed: return "0 ns" - dominant = max(changed, key=lambda index: abs(delta.phases[index])) - threshold = max(1_000_000.0, abs(delta.total) * 0.1) - included = {index for index in changed if abs(delta.phases[index]) >= threshold} + dominant = max(changed, key=lambda index: abs(phases[index])) + threshold = max(1_000_000.0, abs(phases.total) * 0.1) + included = {index for index in changed if abs(phases[index]) >= threshold} included.add(dominant) parts = [ - f"{'◆ ' if index == dominant else ''}{labels[index]} {format_duration(delta.phases[index], signed=True)}" - for index in range(len(labels)) + f"{'◆ ' if index == dominant else ''}{RULESET_PHASES[index].title()} " + f"{format_duration(phases[index], signed=True)}" + for index in range(len(RULESET_PHASES)) if index in included ] if any(index not in included for index in changed): @@ -585,16 +665,14 @@ def report_file_labels(files: Sequence[FileSpec]) -> dict[FileSpec, str]: def format_duration( value_ns: float | None, *, - attribution: bool = False, signed: bool = False, ) -> str: """Format nanoseconds with three significant digits and a local unit.""" if value_ns is None: return NULL - prefix = "!" if attribution and value_ns < 0 else "" divisor, unit = _duration_unit(abs(value_ns)) - return f"{prefix}{_format_scaled(value_ns / divisor, signed=signed)} {unit}" + return f"{_format_scaled(value_ns / divisor, signed=signed)} {unit}" def _format_delta_ms(value_ns: float | None) -> str: @@ -607,16 +685,13 @@ def _format_duration_interval( point_ns: float | None, low_ns: float | None, high_ns: float | None, - *, - attribution: bool = False, ) -> str: if point_ns is None: return NULL if low_ns is None or high_ns is None: - return format_duration(point_ns, attribution=attribution) + return format_duration(point_ns) divisor, unit = _duration_unit(max(abs(point_ns), abs(low_ns), abs(high_ns))) - prefix = "!" if attribution and point_ns < 0 else "" - return f"{prefix}{_format_scaled(low_ns / divisor)}–{_format_scaled(high_ns / divisor)} {unit}" + return f"{_format_scaled(low_ns / divisor)}–{_format_scaled(high_ns / divisor)} {unit}" def _duration_unit(magnitude_ns: float) -> tuple[float, str]: @@ -667,14 +742,7 @@ def _estimate_cell( def _ratio_cell(ratio: RatioEstimate) -> ReportCell: # Retain the point for sorting/filtering while keeping the visible CI cell compact. - tones: dict[ResultClass, CellTone] = { - "higher": "default", - "invalid": "error", - "lower": "positive", - "point_only": "muted", - "unclear": "muted", - } - return text_cell(ratio.estimate.point, format_ratio_summary(ratio), tone=tones[ratio.result_class]) + return text_cell(ratio.estimate.point, format_ratio_summary(ratio), tone=RESULT_TONES[ratio.result_class]) def format_ratio_summary(ratio: RatioEstimate) -> str: @@ -739,14 +807,7 @@ def _result_cell(result_class: ResultClass, issue: str | None, *, rss: bool) -> text = "CI includes 1" else: raise AssertionError(f"unknown result class: {result_class}") - tones: dict[ResultClass, CellTone] = { - "higher": "default", - "invalid": "error", - "lower": "positive", - "point_only": "muted", - "unclear": "muted", - } - return text_cell(result_class, text, tone=tones[result_class]) + return text_cell(result_class, text, tone=RESULT_TONES[result_class]) def _table( diff --git a/benchmarking/reports/render.py b/benchmarking/reports/render.py index b1a7532e..ec30fce0 100644 --- a/benchmarking/reports/render.py +++ b/benchmarking/reports/render.py @@ -37,21 +37,6 @@ } -def report_table(title: str | None, *, caption: str | None = None) -> Table: - """Create one consistently styled Rich report table.""" - - return Table( - title=None if title is None else Text(title, style="bold"), - caption=None if caption is None else Text(caption, style="dim"), - caption_justify="left", - header_style="bold", - box=box.SIMPLE_HEAVY, - expand=True, - collapse_padding=True, - padding=(0, 1), - ) - - def render_rich_table( table_data: ReportTable, *, @@ -60,9 +45,15 @@ def render_rich_table( ) -> Table: """Render one catalog table without interpreting its display strings.""" - table = report_table( - table_data.title if show_title else None, - caption=table_data.caption, + table = Table( + title=Text(table_data.title, style="bold") if show_title else None, + caption=None if table_data.caption is None else Text(table_data.caption, style="dim"), + caption_justify="left", + header_style="bold", + box=box.SIMPLE_HEAVY, + expand=True, + collapse_padding=True, + padding=(0, 1), ) widths: tuple[int | None, ...] = preferred_widths or tuple(None for _ in table_data.columns) for column, preferred_width in zip(table_data.columns, widths, strict=True): diff --git a/benchmarking/reports/store.py b/benchmarking/reports/store.py index 64624688..c639f5dc 100644 --- a/benchmarking/reports/store.py +++ b/benchmarking/reports/store.py @@ -23,25 +23,41 @@ Treatment, ) -type ReportSchemaVersion = Literal[3] -REPORT_SCHEMA_VERSION: Final[ReportSchemaVersion] = 3 +type ReportSchemaVersion = Literal[4] +REPORT_SCHEMA_VERSION: Final[ReportSchemaVersion] = 4 -type TimingSummarySchemaVersion = Literal[3] -TIMING_SUMMARY_SCHEMA_VERSION: Final[TimingSummarySchemaVersion] = 3 +type TimingSummarySchemaVersion = Literal[4] +TIMING_SUMMARY_SCHEMA_VERSION: Final[TimingSummarySchemaVersion] = 4 -class TimingLeafRecord(TypedDict): - """One exclusive timing leaf with unambiguous path segments.""" +type RulesetTimingRole = Literal["program", "equality"] - path: list[str] - ns: int + +class RulesetTimingRecord(TypedDict): + """Exclusive own-work timing for one named ruleset.""" + + name: str + role: RulesetTimingRole + assembly_ns: int + search_ns: int + apply_ns: int + execution_ns: int + merge_ns: int class TimingSummaryRecord(TypedDict): """Versioned engine timing summary embedded in one successful row.""" schema_version: TimingSummarySchemaVersion - timings: list[TimingLeafRecord] + typecheck_ns: int + frontend_parse_ns: int + frontend_other_ns: int + frontend_install_ns: int + commands_actions_ns: int + commands_check_ns: int + commands_other_ns: int + native_rebuild_ns: int + rulesets: list[RulesetTimingRecord] class ReportRecord(TypedDict): diff --git a/egg-math-benchmark/src/main.rs b/egg-math-benchmark/src/main.rs index 188b6299..946b821c 100644 --- a/egg-math-benchmark/src/main.rs +++ b/egg-math-benchmark/src/main.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result, ensure}; use clap::{Parser, ValueEnum}; use egg::{RecExpr, Runner, SimpleScheduler, StopReason}; -use egglog_reports::TimingSummaryV3; +use egglog_reports::{ + PreMergeTiming, ProcessTimings, RulesetTiming, RulesetTimingRole, RunTimings, TimingSummary, +}; use std::{ fs::File, io::BufWriter, @@ -44,7 +46,7 @@ fn main() -> Result<()> { Ok(()) } -fn run_math(proof_mode: ProofMode) -> Result<(TimingSummaryV3, usize)> { +fn run_math(proof_mode: ProofMode) -> Result<(TimingSummary, usize)> { let left: RecExpr = CHECK_LEFT.parse().expect("fixed left check must parse"); let right: RecExpr = CHECK_RIGHT.parse().expect("fixed right check must parse"); let rules = math::rules(); @@ -100,43 +102,36 @@ fn run_math(proof_mode: ProofMode) -> Result<(TimingSummaryV3, usize)> { Duration::ZERO }; - let timing = TimingSummaryV3::new([ - ( - vec!["program".into(), "assembly".into(), String::new()], - Duration::ZERO, - ), - ( - vec!["program".into(), "search".into(), String::new()], - Duration::from_nanos(seconds_to_ns(report.search_time)), - ), - ( - vec!["program".into(), "apply".into(), String::new()], - Duration::from_nanos(seconds_to_ns(report.apply_time)), - ), - ( - vec!["program".into(), "execution".into(), String::new()], - Duration::from_nanos(seconds_to_ns( - (report.total_time - report.search_time - report.apply_time - report.rebuild_time) - .max(0.0), - )), - ), - ( - vec!["program".into(), "merge".into(), String::new()], - Duration::ZERO, - ), - ( - vec!["equality".into(), "rebuild".into(), String::new()], - Duration::from_nanos(seconds_to_ns(report.rebuild_time)), - ), - ( - vec!["commands".into(), "other".into()], - proof_postprocessing, - ), - ]); + let timing = TimingSummary::new( + ProcessTimings { + commands_other: proof_postprocessing, + ..ProcessTimings::default() + }, + RunTimings { + rulesets: vec![RulesetTiming { + name: "".into(), + role: RulesetTimingRole::Program, + assembly: Duration::ZERO, + pre_merge: PreMergeTiming::Split { + search: Duration::from_nanos(seconds_to_ns(report.search_time)), + apply: Duration::from_nanos(seconds_to_ns(report.apply_time)), + unattributed: Duration::from_nanos(seconds_to_ns( + (report.total_time + - report.search_time + - report.apply_time + - report.rebuild_time) + .max(0.0), + )), + }, + merge: Duration::ZERO, + }], + native_rebuild: Duration::from_nanos(seconds_to_ns(report.rebuild_time)), + }, + )?; Ok((timing, report.egraph_nodes)) } -fn write_timing_summary(path: &Path, timing: &TimingSummaryV3) -> Result<()> { +fn write_timing_summary(path: &Path, timing: &TimingSummary) -> Result<()> { let file = File::create(path) .with_context(|| format!("failed to create timing summary {}", path.display()))?; serde_json::to_writer(BufWriter::new(file), timing) diff --git a/egglog-experimental/tests/scheduler_reporting.rs b/egglog-experimental/tests/scheduler_reporting.rs index 36fac13f..178d5c50 100644 --- a/egglog-experimental/tests/scheduler_reporting.rs +++ b/egglog-experimental/tests/scheduler_reporting.rs @@ -15,14 +15,13 @@ const PROGRAM: &str = r#" (seed 1) "#; -fn ruleset_names(report: &RunReport) -> Vec<&str> { - let mut names = report - .ruleset_timings - .keys() - .map(|name| name.as_ref()) - .collect::>(); - names.sort_unstable(); - names +fn ruleset_names(report: &RunReport) -> Vec { + report + .timings() + .rulesets + .iter() + .map(|timing| timing.name.to_string()) + .collect() } #[test] diff --git a/egglog/egglog-reports/src/lib.rs b/egglog/egglog-reports/src/lib.rs index e2b41413..e1135290 100644 --- a/egglog/egglog-reports/src/lib.rs +++ b/egglog/egglog-reports/src/lib.rs @@ -2,6 +2,7 @@ use clap::clap_derive::ValueEnum; use rustc_hash::FxHasher; use serde::Serialize; use std::{ + collections::BTreeMap, fmt::{Display, Formatter}, hash::BuildHasherDefault, sync::Arc, @@ -139,9 +140,47 @@ impl PreMergeTiming { } } -/// Aggregated timing for all iterations of one ruleset. -#[derive(Debug, Serialize, Clone, Copy, PartialEq, Eq, Default)] +/// The semantic responsibility served by a ruleset invocation. +#[derive(Debug, Serialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum RulesetTimingRole { + Program, + Equality, +} + +/// Exclusive process work outside ruleset execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ProcessTimings { + pub typecheck: Duration, + pub frontend_parse: Duration, + pub frontend_other: Duration, + pub frontend_install: Duration, + pub commands_actions: Duration, + pub commands_check: Duration, + pub commands_other: Duration, +} + +impl ProcessTimings { + pub fn total(self) -> Duration { + [ + self.typecheck, + self.frontend_parse, + self.frontend_other, + self.frontend_install, + self.commands_actions, + self.commands_check, + self.commands_other, + ] + .into_iter() + .sum() + } +} + +/// Aggregated own-work timing for all iterations of one ruleset. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct RulesetTiming { + pub name: Arc, + pub role: RulesetTimingRole, /// Building the executable ruleset for each invocation, including lazy /// cached-plan creation on first use. pub assembly: Duration, @@ -149,21 +188,21 @@ pub struct RulesetTiming { pub pre_merge: PreMergeTiming, /// Resolving and installing staged updates. pub merge: Duration, - /// Rebuilding indexes and e-graph state after merge. - pub rebuild: Duration, } impl RulesetTiming { - pub fn total(self) -> Duration { - self.assembly + self.pre_merge.total() + self.merge + self.rebuild + fn add_iteration(&mut self, iteration: &IterationReport) { + self.assembly += iteration.assembly_time; + self.pre_merge.union(iteration.rule_set_report.pre_merge); + self.merge += iteration.rule_set_report.merge_time; } +} - fn union(&mut self, other: Self) { - self.assembly += other.assembly; - self.pre_merge.union(other.pre_merge); - self.merge += other.merge; - self.rebuild += other.rebuild; - } +/// Derived timing view over a run's annotated iterations. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct RunTimings { + pub rulesets: Vec, + pub native_rebuild: Duration, } impl RuleSetReport { @@ -212,6 +251,14 @@ impl IterationReport { } } +/// One ruleset invocation and the responsibility it served when it ran. +#[derive(Debug, Serialize, Clone)] +pub struct RulesetIteration { + pub name: Arc, + pub role: RulesetTimingRole, + pub report: Arc, +} + /// Running a schedule produces a report of the results. /// This includes rough timing information and whether /// the database was updated. @@ -219,17 +266,15 @@ impl IterationReport { /// information together. #[derive(Debug, Serialize, Clone)] pub struct RunReport { - // Since `IterationReport`s are immutable, we can reference count them to avoid - // expensive cloning when e-graphs are cloned. - pub iterations: Vec>, + // Since iteration reports are immutable, they are reference counted to + // avoid expensive cloning when e-graphs are cloned. + pub iterations: Vec, /// If any changes were made to the database. pub updated: bool, /// True if this run observed no database changes and there is no deferred /// scheduler work requiring another iteration. pub can_stop: bool, - pub search_and_apply_time_per_rule: HashMap, Duration>, pub num_matches_per_rule: HashMap, usize>, - pub ruleset_timings: HashMap, RulesetTiming>, } impl Default for RunReport { @@ -238,16 +283,15 @@ impl Default for RunReport { iterations: Vec::new(), updated: false, can_stop: true, - search_and_apply_time_per_rule: HashMap::default(), num_matches_per_rule: HashMap::default(), - ruleset_timings: HashMap::default(), } } } impl Display for RunReport { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut rule_times_vec: Vec<_> = self.search_and_apply_time_per_rule.iter().collect(); + let rule_times = self.search_and_apply_time_per_rule(); + let mut rule_times_vec: Vec<_> = rule_times.iter().collect(); rule_times_vec.sort_by_key(|(_, time)| **time); for (rule, time) in rule_times_vec { @@ -260,10 +304,10 @@ impl Display for RunReport { )?; } - for (ruleset, timing) in &self.ruleset_timings { + let timings = self.timings(); + for timing in &timings.rulesets { let assembly_time = timing.assembly.as_secs_f64(); let merge_time = timing.merge.as_secs_f64(); - let rebuild_time = timing.rebuild.as_secs_f64(); match timing.pre_merge { PreMergeTiming::Split { search, @@ -272,7 +316,8 @@ impl Display for RunReport { } => { writeln!( f, - "Ruleset {ruleset}: assembly {assembly_time:.3}s, search {:.3}s, apply {:.3}s, unattributed {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", + "Ruleset {}: assembly {assembly_time:.3}s, search {:.3}s, apply {:.3}s, unattributed {:.3}s, merge {merge_time:.3}s", + timing.name, search.as_secs_f64(), apply.as_secs_f64(), unattributed.as_secs_f64(), @@ -281,12 +326,18 @@ impl Display for RunReport { PreMergeTiming::Combined { elapsed } => { writeln!( f, - "Ruleset {ruleset}: assembly {assembly_time:.3}s, pre-merge {:.3}s, merge {merge_time:.3}s, rebuild {rebuild_time:.3}s", + "Ruleset {}: assembly {assembly_time:.3}s, pre-merge {:.3}s, merge {merge_time:.3}s", + timing.name, elapsed.as_secs_f64(), )?; } } } + writeln!( + f, + "Native rebuild: {:.3}s", + timings.native_rebuild.as_secs_f64() + )?; Ok(()) } @@ -305,61 +356,80 @@ impl RunReport { s } - fn union_times( - times: &mut HashMap, Duration>, - other_times: HashMap, Duration>, - ) { - for (k, v) in other_times { - *times.entry(k).or_default() += v; - } - } - fn union_counts(counts: &mut HashMap, usize>, other_counts: HashMap, usize>) { for (k, v) in other_counts { *counts.entry(k).or_default() += v; } } - pub fn singleton(ruleset: &str, iteration: IterationReport) -> Self { + pub fn singleton(ruleset: &str, role: RulesetTimingRole, iteration: IterationReport) -> Self { let mut report = RunReport::default(); for rule in iteration.rules() { - *report - .search_and_apply_time_per_rule - .entry(rule.clone()) - .or_default() += iteration.rule_set_report.rule_search_and_apply_time(rule); *report.num_matches_per_rule.entry(rule.clone()).or_default() += iteration.rule_set_report.num_matches(rule); } - let ruleset: Arc = ruleset.into(); - report.ruleset_timings.insert( - ruleset, - RulesetTiming { - assembly: iteration.assembly_time, - pre_merge: iteration.rule_set_report.pre_merge, - merge: iteration.rule_set_report.merge_time, - rebuild: iteration.rebuild_time, - }, - ); report.updated = iteration.changed(); report.can_stop = !report.updated; - report.iterations.push(Arc::new(iteration)); + report.iterations.push(RulesetIteration { + name: ruleset.into(), + role, + report: Arc::new(iteration), + }); report } - pub fn add_iteration(&mut self, ruleset: &str, iteration: IterationReport) { - self.union(RunReport::singleton(ruleset, iteration)); + pub fn add_iteration( + &mut self, + ruleset: &str, + role: RulesetTimingRole, + iteration: IterationReport, + ) { + self.union(RunReport::singleton(ruleset, role, iteration)); } - /// Total wall-clock work recorded by all ruleset phase timers. - pub fn total_ruleset_time(&self) -> Duration { - self.ruleset_timings - .values() - .copied() - .map(RulesetTiming::total) - .sum() + /// Derive per-rule search-and-apply totals from the recorded iterations. + pub fn search_and_apply_time_per_rule(&self) -> HashMap, Duration> { + let mut result = HashMap::default(); + for iteration in &self.iterations { + for rule in iteration.report.rules() { + *result.entry(rule.clone()).or_default() += iteration + .report + .rule_set_report + .rule_search_and_apply_time(rule); + } + } + result + } + + /// Derive the ruleset-own-work and global rebuild partition of this run. + pub fn timings(&self) -> RunTimings { + let mut rulesets = BTreeMap::<(RulesetTimingRole, Arc), RulesetTiming>::new(); + let mut native_rebuild = Duration::ZERO; + for iteration in &self.iterations { + native_rebuild = native_rebuild.saturating_add(iteration.report.rebuild_time); + let key = (iteration.role, iteration.name.clone()); + match rulesets.entry(key) { + std::collections::btree_map::Entry::Occupied(mut entry) => { + entry.get_mut().add_iteration(&iteration.report); + } + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(RulesetTiming { + name: iteration.name.clone(), + role: iteration.role, + assembly: iteration.report.assembly_time, + pre_merge: iteration.report.rule_set_report.pre_merge, + merge: iteration.report.rule_set_report.merge_time, + }); + } + } + } + RunTimings { + rulesets: rulesets.into_values().collect(), + native_rebuild, + } } /// Merge two reports. @@ -367,86 +437,113 @@ impl RunReport { self.iterations.extend(other.iterations); self.updated |= other.updated; self.can_stop &= other.can_stop; - RunReport::union_times( - &mut self.search_and_apply_time_per_rule, - other.search_and_apply_time_per_rule, - ); RunReport::union_counts(&mut self.num_matches_per_rule, other.num_matches_per_rule); - for (ruleset, timing) in other.ruleset_timings { - self.ruleset_timings - .entry(ruleset) - .and_modify(|current| current.union(timing)) - .or_insert(timing); - } } } -/// One exclusive timing leaf in the benchmark transport. -/// -/// Static mechanism and phase names occupy the first two segments. Ruleset -/// leaves add the exact ruleset name as a third segment. Segments are stored -/// separately so user names containing `/` remain unambiguous. +/// Compact timing for one ruleset in the benchmark transport. #[derive(Debug, Serialize, Clone, PartialEq, Eq)] -pub struct TimingLeafV3 { - pub path: Vec, - pub ns: u64, +pub struct RulesetTimingSummary { + pub name: String, + pub role: RulesetTimingRole, + pub assembly_ns: u64, + pub search_ns: u64, + pub apply_ns: u64, + pub execution_ns: u64, + pub merge_ns: u64, } /// Versioned, deterministic timing transport for successful egglog runs. /// -/// The values are exclusive wall-clock leaves: their sum can be subtracted -/// once from process wall time to derive residual. Parent totals are never -/// stored. Construction sorts paths lexicographically, rejects duplicate paths -/// as a producer bug, saturates nanoseconds at [`u64::MAX`], and never -/// truncates the leaf list. +/// Every value is an exclusive wall-clock leaf. Rulesets are sorted by semantic +/// role and name. Native rebuild is global because the ruleset whose tail +/// happened to flush updates is not its semantic owner. #[derive(Debug, Serialize, Clone, PartialEq, Eq)] -pub struct TimingSummaryV3 { +pub struct TimingSummary { pub schema_version: u32, - pub timings: Vec, + pub typecheck_ns: u64, + pub frontend_parse_ns: u64, + pub frontend_other_ns: u64, + pub frontend_install_ns: u64, + pub commands_actions_ns: u64, + pub commands_check_ns: u64, + pub commands_other_ns: u64, + pub native_rebuild_ns: u64, + pub rulesets: Vec, } -/// A requested timing summary contains a ruleset whose split phase timing was -/// not recorded. +/// A requested timing summary cannot satisfy the serial, single-role contract. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct PhaseTimingUnavailable { - pub ruleset: String, +pub enum TimingSummaryError { + PhaseTimingUnavailable { ruleset: String }, + InconsistentRulesetRole { ruleset: String }, } -impl Display for PhaseTimingUnavailable { +impl Display for TimingSummaryError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "split pre-merge timing is unavailable for ruleset {ruleset:?}", - ruleset = self.ruleset, - ) + match self { + Self::PhaseTimingUnavailable { ruleset } => write!( + f, + "split pre-merge timing is unavailable for ruleset {ruleset:?}" + ), + Self::InconsistentRulesetRole { ruleset } => { + write!(f, "ruleset {ruleset:?} ran with inconsistent timing roles") + } + } } } -impl std::error::Error for PhaseTimingUnavailable {} - -impl TimingSummaryV3 { - pub fn new(timings: impl IntoIterator, Duration)>) -> Self { - let mut timings = timings - .into_iter() - .map(|(path, duration)| TimingLeafV3 { - path, - ns: duration_ns(duration), - }) - .collect::>(); - assert!( - timings.iter().all(|timing| !timing.path.is_empty()), - "timing paths must not be empty" - ); - timings.sort_unstable_by(|left, right| left.path.cmp(&right.path)); - assert!( - timings.windows(2).all(|pair| pair[0].path != pair[1].path), - "duplicate timing path" - ); - - Self { - schema_version: 3, - timings, +impl std::error::Error for TimingSummaryError {} + +impl TimingSummary { + pub fn new(process: ProcessTimings, mut run: RunTimings) -> Result { + run.rulesets.sort_unstable_by(|left, right| { + (left.role, &left.name).cmp(&(right.role, &right.name)) + }); + let mut roles = BTreeMap::new(); + let mut rulesets = Vec::with_capacity(run.rulesets.len()); + for timing in run.rulesets { + if roles + .insert(timing.name.clone(), timing.role) + .is_some_and(|role| role != timing.role) + { + return Err(TimingSummaryError::InconsistentRulesetRole { + ruleset: timing.name.to_string(), + }); + } + let PreMergeTiming::Split { + search, + apply, + unattributed, + } = timing.pre_merge + else { + return Err(TimingSummaryError::PhaseTimingUnavailable { + ruleset: timing.name.to_string(), + }); + }; + rulesets.push(RulesetTimingSummary { + name: timing.name.to_string(), + role: timing.role, + assembly_ns: duration_ns(timing.assembly), + search_ns: duration_ns(search), + apply_ns: duration_ns(apply), + execution_ns: duration_ns(unattributed), + merge_ns: duration_ns(timing.merge), + }); } + + Ok(Self { + schema_version: 4, + typecheck_ns: duration_ns(process.typecheck), + frontend_parse_ns: duration_ns(process.frontend_parse), + frontend_other_ns: duration_ns(process.frontend_other), + frontend_install_ns: duration_ns(process.frontend_install), + commands_actions_ns: duration_ns(process.commands_actions), + commands_check_ns: duration_ns(process.commands_check), + commands_other_ns: duration_ns(process.commands_other), + native_rebuild_ns: duration_ns(run.native_rebuild), + rulesets, + }) } } @@ -466,57 +563,12 @@ mod tests { } } - #[test] - fn timing_summary_v3_exact_json_is_sorted_and_segmented() { - let summary = TimingSummaryV3::new([ - ( - vec!["program".into(), "search".into(), "rules/λ".into()], - Duration::new(1, 234), - ), - ( - vec!["commands".into(), "check".into()], - Duration::from_nanos(6), - ), - ( - vec!["equality".into(), "rebuild".into(), "rules/λ".into()], - Duration::from_nanos(67), - ), - ( - vec!["frontend".into(), "parse".into()], - Duration::from_nanos(1), - ), - ( - vec!["typecheck".into(), "total".into()], - Duration::from_nanos(2), - ), - ]); - let json = serde_json::to_string(&summary).unwrap(); - - assert_eq!( - json, - r#"{"schema_version":3,"timings":[{"path":["commands","check"],"ns":6},{"path":["equality","rebuild","rules/λ"],"ns":67},{"path":["frontend","parse"],"ns":1},{"path":["program","search","rules/λ"],"ns":1000000234},{"path":["typecheck","total"],"ns":2}]}"# - ); - } - - #[test] - fn timing_summary_v3_empty_report_golden() { - let summary = TimingSummaryV3::new([]); - let json = serde_json::to_string(&summary).unwrap(); - - assert_eq!(json, r#"{"schema_version":3,"timings":[]}"#); - } - - #[test] - #[should_panic(expected = "timing paths must not be empty")] - fn timing_summary_v3_rejects_an_empty_path() { - TimingSummaryV3::new([(vec![], Duration::ZERO)]); - } - #[test] fn run_report_aggregates_every_iteration_of_a_ruleset() { let mut report = RunReport::default(); report.add_iteration( "timed", + RulesetTimingRole::Program, IterationReport { rule_set_report: RuleSetReport { pre_merge: split(11, 7, 3), @@ -529,6 +581,7 @@ mod tests { ); report.add_iteration( "timed", + RulesetTimingRole::Program, IterationReport { rule_set_report: RuleSetReport { pre_merge: split(19, 5, 4), @@ -540,97 +593,134 @@ mod tests { }, ); + let timings = report.timings(); assert_eq!( - report.ruleset_timings["timed"].pre_merge.total(), + timings.rulesets[0].pre_merge.total(), Duration::from_nanos(49) ); - assert_eq!( - report.ruleset_timings["timed"].total(), - Duration::from_nanos(136) - ); + assert_eq!(timings.rulesets[0].assembly, Duration::from_nanos(5)); + assert_eq!(timings.rulesets[0].merge, Duration::from_nanos(36)); + assert_eq!(timings.native_rebuild, Duration::from_nanos(46)); } #[test] - fn timing_summary_v3_does_not_truncate_leaves() { - let summary = TimingSummaryV3::new((0..40).rev().map(|index| { - ( - vec![ - "program".into(), - "search".into(), - format!("ruleset-{index:02}"), + fn timing_summary_exact_json_is_dense_and_sorted() { + let summary = TimingSummary::new( + ProcessTimings { + typecheck: Duration::from_nanos(2), + frontend_parse: Duration::from_nanos(1), + commands_check: Duration::from_nanos(6), + ..ProcessTimings::default() + }, + RunTimings { + rulesets: vec![ + RulesetTiming { + name: "@parent".into(), + role: RulesetTimingRole::Equality, + assembly: Duration::from_nanos(8), + pre_merge: split(9, 10, 11), + merge: Duration::from_nanos(12), + }, + RulesetTiming { + name: "rules/λ".into(), + role: RulesetTimingRole::Program, + assembly: Duration::ZERO, + pre_merge: split(1_000_000_234, 3, 4), + merge: Duration::from_nanos(5), + }, ], - Duration::from_nanos(index + 1), - ) - })); + native_rebuild: Duration::from_nanos(13), + }, + ) + .unwrap(); - assert_eq!(summary.timings.len(), 40); - assert_eq!(summary.timings.first().unwrap().path[2], "ruleset-00"); - assert_eq!(summary.timings.last().unwrap().path[2], "ruleset-39"); + assert_eq!( + serde_json::to_string(&summary).unwrap(), + r#"{"schema_version":4,"typecheck_ns":2,"frontend_parse_ns":1,"frontend_other_ns":0,"frontend_install_ns":0,"commands_actions_ns":0,"commands_check_ns":6,"commands_other_ns":0,"native_rebuild_ns":13,"rulesets":[{"name":"rules/λ","role":"program","assembly_ns":0,"search_ns":1000000234,"apply_ns":3,"execution_ns":4,"merge_ns":5},{"name":"@parent","role":"equality","assembly_ns":8,"search_ns":9,"apply_ns":10,"execution_ns":11,"merge_ns":12}]}"# + ); } #[test] - fn timing_summary_v3_saturates_nanoseconds_to_u64() { - let summary = TimingSummaryV3::new([( - vec!["program".into(), "search".into(), "long".into()], - Duration::from_secs(u64::MAX), - )]); - - assert_eq!(summary.timings[0].ns, u64::MAX); + fn timing_summary_empty_report_golden() { + let summary = TimingSummary::new(ProcessTimings::default(), RunTimings::default()).unwrap(); + assert_eq!( + serde_json::to_string(&summary).unwrap(), + r#"{"schema_version":4,"typecheck_ns":0,"frontend_parse_ns":0,"frontend_other_ns":0,"frontend_install_ns":0,"commands_actions_ns":0,"commands_check_ns":0,"commands_other_ns":0,"native_rebuild_ns":0,"rulesets":[]}"# + ); } #[test] - #[should_panic(expected = "duplicate timing path")] - fn timing_summary_v3_rejects_duplicate_paths() { - TimingSummaryV3::new([ - (vec!["commands".into(), "check".into()], Duration::ZERO), - (vec!["commands".into(), "check".into()], Duration::ZERO), - ]); + fn timing_summary_does_not_truncate_rulesets_and_saturates_nanoseconds() { + let summary = TimingSummary::new( + ProcessTimings::default(), + RunTimings { + rulesets: (0..40) + .rev() + .map(|index| RulesetTiming { + name: format!("ruleset-{index:02}").into(), + role: RulesetTimingRole::Program, + assembly: Duration::ZERO, + pre_merge: split(index + 1, 0, 0), + merge: Duration::ZERO, + }) + .collect(), + native_rebuild: Duration::from_secs(u64::MAX), + }, + ) + .unwrap(); + + assert_eq!(summary.rulesets.len(), 40); + assert_eq!(summary.rulesets.first().unwrap().name, "ruleset-00"); + assert_eq!(summary.rulesets.last().unwrap().name, "ruleset-39"); + assert_eq!(summary.native_rebuild_ns, u64::MAX); } #[test] - fn combined_iteration_degrades_aggregated_pre_merge_timing() { - let mut report = RunReport::default(); - report.add_iteration( - "mixed", - IterationReport { - rule_set_report: RuleSetReport { - pre_merge: split(1, 2, 3), - merge_time: Duration::from_nanos(7), - ..RuleSetReport::default() - }, - rebuild_time: Duration::from_nanos(11), - assembly_time: Duration::from_nanos(2), - }, - ); - report.add_iteration( - "mixed", - IterationReport { - rule_set_report: RuleSetReport { + fn timing_summary_rejects_combined_timing_and_inconsistent_roles() { + let combined = TimingSummary::new( + ProcessTimings::default(), + RunTimings { + rulesets: vec![RulesetTiming { + name: "mixed".into(), + role: RulesetTimingRole::Program, + assembly: Duration::ZERO, pre_merge: PreMergeTiming::Combined { elapsed: Duration::from_nanos(5), }, - merge_time: Duration::from_nanos(13), - ..RuleSetReport::default() - }, - rebuild_time: Duration::from_nanos(17), - assembly_time: Duration::from_nanos(3), + merge: Duration::ZERO, + }], + native_rebuild: Duration::ZERO, }, ); - assert_eq!( - report.ruleset_timings["mixed"], - RulesetTiming { - assembly: Duration::from_nanos(5), - pre_merge: PreMergeTiming::Combined { - elapsed: Duration::from_nanos(11), - }, - merge: Duration::from_nanos(20), - rebuild: Duration::from_nanos(28), - } + combined, + Err(TimingSummaryError::PhaseTimingUnavailable { + ruleset: "mixed".into() + }) + ); + + let duplicate = |role| RulesetTiming { + name: "mixed".into(), + role, + assembly: Duration::ZERO, + pre_merge: split(0, 0, 0), + merge: Duration::ZERO, + }; + let inconsistent = TimingSummary::new( + ProcessTimings::default(), + RunTimings { + rulesets: vec![ + duplicate(RulesetTimingRole::Program), + duplicate(RulesetTimingRole::Equality), + ], + native_rebuild: Duration::ZERO, + }, ); assert_eq!( - report.ruleset_timings["mixed"].total(), - Duration::from_nanos(64) + inconsistent, + Err(TimingSummaryError::InconsistentRulesetRole { + ruleset: "mixed".into() + }) ); } } diff --git a/egglog/src/ast/mod.rs b/egglog/src/ast/mod.rs index 6b526846..5f70e7de 100644 --- a/egglog/src/ast/mod.rs +++ b/egglog/src/ast/mod.rs @@ -61,7 +61,13 @@ pub struct ProofConstructorNames { #[derive(Clone, Debug)] /// The egglog internal representation of already compiled rules -pub(crate) enum Ruleset { +pub(crate) struct Ruleset { + pub kind: RulesetKind, + pub timing_role: egglog_reports::RulesetTimingRole, +} + +#[derive(Clone, Debug)] +pub(crate) enum RulesetKind { /// Represents a ruleset with a set of rules. Rules(IndexMap), /// A combined ruleset may contain other rulesets. diff --git a/egglog/src/lib.rs b/egglog/src/lib.rs index 34439bc1..c4b47826 100644 --- a/egglog/src/lib.rs +++ b/egglog/src/lib.rs @@ -8,7 +8,6 @@ pub mod constraint; mod core; mod exec_state; pub mod extract; -mod phase_timers; pub mod prelude; mod proofs; @@ -44,7 +43,7 @@ use egglog_bridge::{ColumnTy, QueryEntry}; use egglog_core_relations as core_relations; use egglog_numeric_id as numeric_id; use egglog_reports::{ - PhaseTimingUnavailable, PreMergeTiming, ReportLevel, RunReport, TimingSummaryV3, + ProcessTimings, ReportLevel, RulesetTimingRole, RunReport, TimingSummary, TimingSummaryError, }; pub use exec_state::{ Context, Core, Enode, FullState, FunctionEntry, PureState, Read, ReadState, Write, WriteState, @@ -322,19 +321,14 @@ pub struct EGraph { pushed_egraph: Option>, functions: IndexMap, rulesets: IndexMap, - /// The semantic responsibility of each declared ruleset's execution. - ruleset_timing_roles: IndexMap, pub fact_directory: Option, pub seminaive: bool, pub no_decomp: bool, type_info: TypeInfo, /// The run report unioned over all runs so far. overall_run_report: RunReport, - /// Roles for every ruleset present in `overall_run_report`, including work - /// performed inside a scope that has since been popped. - overall_ruleset_timing_roles: IndexMap, /// Exclusive process work outside ruleset execution. - phase_timings: phase_timers::PhaseTimings, + process_timings: ProcessTimings, schedulers: DenseIdMap, commands: IndexMap>, extension_state: HashMap>, @@ -450,13 +444,11 @@ impl EGraph { pushed_egraph: Default::default(), functions: Default::default(), rulesets: Default::default(), - ruleset_timing_roles: Default::default(), fact_directory: None, seminaive: true, no_decomp: false, overall_run_report: Default::default(), - overall_ruleset_timing_roles: Default::default(), - phase_timings: Default::default(), + process_timings: Default::default(), type_info: Default::default(), schedulers: Default::default(), commands: Default::default(), @@ -576,10 +568,13 @@ impl EGraph { None, ); - eg.rulesets - .insert("".into(), Ruleset::Rules(Default::default())); - eg.ruleset_timing_roles - .insert("".into(), phase_timers::RulesetTimingRole::Program); + eg.rulesets.insert( + "".into(), + Ruleset { + kind: RulesetKind::Rules(Default::default()), + timing_role: RulesetTimingRole::Program, + }, + ); // The generic `get-fresh!` mint primitive is registered on every e-graph. // Doing it here — rather than per-eq-sort — means it is present whenever @@ -857,12 +852,8 @@ impl EGraph { Some(mut e) => { // Preserve the overall report from the popped egraph std::mem::swap(&mut self.overall_run_report, &mut e.overall_run_report); - std::mem::swap( - &mut self.overall_ruleset_timing_roles, - &mut e.overall_ruleset_timing_roles, - ); // Work performed in the popped scope still belongs to this run. - std::mem::swap(&mut self.phase_timings, &mut e.phase_timings); + std::mem::swap(&mut self.process_timings, &mut e.process_timings); // Preserve the symbol generator so that fresh symbols // generated after pop don't collide with ones generated before pop. std::mem::swap(&mut self.parser.symbol_gen, &mut e.parser.symbol_gen); @@ -1389,13 +1380,13 @@ impl EGraph { rulesets: &IndexMap, ids: &mut Vec, ) { - match &rulesets[ruleset] { - Ruleset::Rules(rules) => { + match &rulesets[ruleset].kind { + RulesetKind::Rules(rules) => { for (_, id) in rules.values() { ids.push(*id); } } - Ruleset::Combined(sub_rulesets) => { + RulesetKind::Combined(sub_rulesets) => { for sub_ruleset in sub_rulesets { collect_rule_ids(sub_ruleset, rulesets, ids); } @@ -1411,26 +1402,15 @@ impl EGraph { .run_rules(&rule_ids) .map_err(|e| Error::BackendError(e.to_string()))?; - let report = RunReport::singleton(ruleset, iteration_report); - self.record_ruleset_timing_role(ruleset); + let report = RunReport::singleton( + ruleset, + self.rulesets[ruleset].timing_role, + iteration_report, + ); self.overall_run_report.union(report.clone()); Ok(report) } - fn record_ruleset_timing_role(&mut self, ruleset: &str) { - let role = self.ruleset_timing_roles[ruleset]; - match self.overall_ruleset_timing_roles.entry(ruleset.to_owned()) { - Entry::Occupied(entry) => assert_eq!( - *entry.get(), - role, - "a ruleset's timing role changed after it was recorded" - ), - Entry::Vacant(entry) => { - entry.insert(role); - } - } - } - fn add_rule(&mut self, rule: ast::ResolvedRule) -> Result { // The `:naive` rule option opts a single rule out of seminaive // evaluation. This widens primitive-context selection from @@ -1451,9 +1431,13 @@ impl EGraph { // it expects only `union` on constructors (not set). let union_to_set = self.proof_state.original_typechecking.is_none(); - match self.rulesets.get(&rule.ruleset) { - Some(Ruleset::Rules(_)) => {} - Some(Ruleset::Combined(_)) => { + match self + .rulesets + .get(&rule.ruleset) + .map(|ruleset| &ruleset.kind) + { + Some(RulesetKind::Rules(_)) => {} + Some(RulesetKind::Combined(_)) => { return Err(Error::CombinedRulesetError( rule.ruleset.clone(), rule.span.clone(), @@ -1486,7 +1470,11 @@ impl EGraph { translator.build(no_decomp) }; - let Some(Ruleset::Rules(rules)) = self.rulesets.get_mut(&rule.ruleset) else { + let Some(Ruleset { + kind: RulesetKind::Rules(rules), + .. + }) = self.rulesets.get_mut(&rule.ruleset) + else { unreachable!("ruleset was validated before compiling the rule") }; match rules.entry(rule.name.clone()) { @@ -1918,9 +1906,9 @@ impl EGraph { let mut timing_role = None; for ruleset in &rulesets { let role = self - .ruleset_timing_roles + .rulesets .get(ruleset) - .copied() + .map(|ruleset| ruleset.timing_role) .ok_or_else(|| Error::NoSuchRuleset(ruleset.clone(), span.clone()))?; if timing_role.is_some_and(|expected| expected != role) { return Err(Error::MixedRulesetResponsibilities(name, span.clone())); @@ -1929,12 +1917,11 @@ impl EGraph { } match self.rulesets.entry(name.clone()) { Entry::Occupied(_) => panic!("Ruleset '{name}' was already present"), - Entry::Vacant(e) => e.insert(Ruleset::Combined(rulesets)), + Entry::Vacant(e) => e.insert(Ruleset { + kind: RulesetKind::Combined(rulesets), + timing_role: timing_role.unwrap_or(RulesetTimingRole::Program), + }), }; - self.ruleset_timing_roles.insert( - name, - timing_role.unwrap_or(phase_timers::RulesetTimingRole::Program), - ); Ok(()) } @@ -1949,15 +1936,17 @@ impl EGraph { .iter() .any(|generated| generated.as_str() == name) { - phase_timers::RulesetTimingRole::EqualityMaintenance + RulesetTimingRole::Equality } else { - phase_timers::RulesetTimingRole::Program + RulesetTimingRole::Program }; match self.rulesets.entry(name.clone()) { Entry::Occupied(_) => panic!("Ruleset '{name}' was already present"), - Entry::Vacant(e) => e.insert(Ruleset::Rules(Default::default())), + Entry::Vacant(e) => e.insert(Ruleset { + kind: RulesetKind::Rules(Default::default()), + timing_role, + }), }; - self.ruleset_timing_roles.insert(name, timing_role); } fn check_facts(&mut self, span: &Span, facts: &[ResolvedFact]) -> Result<(), Error> { @@ -2008,8 +1997,7 @@ impl EGraph { self.backend.free_rule(id); self.backend.free_external_func(ext_id); let iteration_report = run_result.map_err(|e| Error::BackendError(e.to_string()))?; - self.phase_timings - .add(phase_timers::COMMANDS_CHECK, iteration_report.total_time()); + self.process_timings.commands_check += iteration_report.total_time(); let ext_sc_val = ext_sc.lock().unwrap().take(); let matched = matches!(ext_sc_val, Some(())); @@ -2046,30 +2034,22 @@ impl EGraph { _ => CommandPhase::Other, }; let command_timer = Instant::now(); - let process_before = self.phase_timings.total(); - let ruleset_before = self.overall_run_report.total_ruleset_time(); + let process_before = self.process_timings.total(); + let iteration_before = self.overall_run_report.iterations.len(); let result = self.run_command_inner(command); - let nested_process = self.phase_timings.total().saturating_sub(process_before); - let nested_rulesets = self - .overall_run_report - .total_ruleset_time() - .saturating_sub(ruleset_before); + let nested_process = self.process_timings.total().saturating_sub(process_before); + let nested_rulesets = self.overall_run_report.iterations[iteration_before..] + .iter() + .map(|iteration| iteration.report.total_time()) + .sum(); let own_time = command_timer .elapsed() .saturating_sub(nested_process + nested_rulesets); match phase { - CommandPhase::Install => self - .phase_timings - .add(phase_timers::FRONTEND_INSTALL, own_time), - CommandPhase::Actions => self - .phase_timings - .add(phase_timers::COMMANDS_ACTIONS, own_time), - CommandPhase::Check => self - .phase_timings - .add(phase_timers::COMMANDS_CHECK, own_time), - CommandPhase::Other => self - .phase_timings - .add(phase_timers::COMMANDS_OTHER, own_time), + CommandPhase::Install => self.process_timings.frontend_install += own_time, + CommandPhase::Actions => self.process_timings.commands_actions += own_time, + CommandPhase::Check => self.process_timings.commands_check += own_time, + CommandPhase::Other => self.process_timings.commands_other += own_time, } result } @@ -2649,8 +2629,7 @@ impl EGraph { // TODO this is ugly- we don't need an entire e-graph just for type information. let typecheck_timer = Instant::now(); let typechecked = original_typechecking.typecheck_program(&desugared)?; - self.phase_timings - .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); + self.process_timings.typecheck += typecheck_timer.elapsed(); for command in &typechecked { if let Err(reason) = command_supports_proof_encoding( @@ -2669,8 +2648,7 @@ impl EGraph { } else { let typecheck_timer = Instant::now(); let mut typechecked = self.typecheck_program(&desugared)?; - self.phase_timings - .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); + self.process_timings.typecheck += typecheck_timer.elapsed(); typechecked = remove_globals::remove_globals(typechecked, &mut self.parser.symbol_gen); for command in &typechecked { @@ -2685,13 +2663,10 @@ impl EGraph { /// When will_run is true, adds to `desugared_commands_run_so_far`, which is used for proof checking. fn resolve_command(&mut self, command: Command) -> Result { let lowering_timer = Instant::now(); - let nested_before = self.phase_timings.total(); + let nested_before = self.process_timings.total(); let resolved = self.resolve_command_inner(command); - let nested = self.phase_timings.total().saturating_sub(nested_before); - self.phase_timings.add( - phase_timers::FRONTEND_OTHER, - lowering_timer.elapsed().saturating_sub(nested), - ); + let nested = self.process_timings.total().saturating_sub(nested_before); + self.process_timings.frontend_other += lowering_timer.elapsed().saturating_sub(nested); resolved } @@ -2745,8 +2720,7 @@ impl EGraph { // Now typecheck using self, adding term type information. let typecheck_timer = Instant::now(); let desugared_typechecked = self.typecheck_program(&desugared)?; - self.phase_timings - .add(phase_timers::TYPECHECK, typecheck_timer.elapsed()); + self.process_timings.typecheck += typecheck_timer.elapsed(); // Remove the globals the term encoding itself introduced (its minted // `let`s), the same way source-level globals were removed above. let desugared_typechecked = remove_globals::remove_globals( @@ -2789,8 +2763,7 @@ impl EGraph { &mut self.parser.symbol_gen, macro_type_info, ); - self.phase_timings - .add(phase_timers::FRONTEND_OTHER, macro_timer.elapsed()); + self.process_timings.frontend_other += macro_timer.elapsed(); let macro_expanded = macro_expanded?; for command in macro_expanded { @@ -2799,8 +2772,7 @@ impl EGraph { let include_timer = Instant::now(); let s = std::fs::read_to_string(file) .map_err(|e| Error::IoError(file.clone().into(), e, span.clone())); - self.phase_timings - .add(phase_timers::FRONTEND_OTHER, include_timer.elapsed()); + self.process_timings.frontend_other += include_timer.elapsed(); let s = s?; let included_program = self.parse_program_timed(Some(file.clone()), &s)?; // run program internal on these include commands @@ -2893,8 +2865,7 @@ impl EGraph { ) -> Result, Error> { let parse_timer = Instant::now(); let parsed = self.parser.get_program_from_string(filename, input); - self.phase_timings - .add(phase_timers::FRONTEND_PARSE, parse_timer.elapsed()); + self.process_timings.frontend_parse += parse_timer.elapsed(); Ok(parsed?) } @@ -2952,53 +2923,8 @@ impl EGraph { &self.overall_run_report } - pub(crate) fn timing_summary(&self) -> Result { - let mut leaves = self.phase_timings.timing_leaves(); - for (ruleset, timing) in &self.overall_run_report.ruleset_timings { - let PreMergeTiming::Split { - search, - apply, - unattributed, - } = timing.pre_merge - else { - return Err(PhaseTimingUnavailable { - ruleset: ruleset.to_string(), - }); - }; - let role = self - .overall_ruleset_timing_roles - .get(ruleset.as_ref()) - .unwrap_or_else(|| panic!("missing timing role for ruleset {ruleset:?}")); - let responsibility = match role { - phase_timers::RulesetTimingRole::Program => "program", - phase_timers::RulesetTimingRole::EqualityMaintenance => "equality", - }; - for (phase, duration) in [ - ("assembly", timing.assembly), - ("search", search), - ("apply", apply), - ("execution", unattributed), - ("merge", timing.merge), - ] { - leaves.push(( - vec![ - responsibility.to_owned(), - phase.to_owned(), - ruleset.to_string(), - ], - duration, - )); - } - leaves.push(( - vec![ - "equality".to_owned(), - "rebuild".to_owned(), - ruleset.to_string(), - ], - timing.rebuild, - )); - } - Ok(TimingSummaryV3::new(leaves)) + pub(crate) fn timing_summary(&self) -> Result { + TimingSummary::new(self.process_timings, self.overall_run_report.timings()) } /// Convert from an egglog value to a Rust type. @@ -3203,12 +3129,15 @@ impl EGraph { // Tear the temporary rule + ruleset down whether the body // succeeded or not. - if let Some(Ruleset::Rules(rules)) = self.rulesets.swap_remove(&ruleset) { + if let Some(Ruleset { + kind: RulesetKind::Rules(rules), + .. + }) = self.rulesets.swap_remove(&ruleset) + { for (_, rule) in rules { self.backend.free_rule(rule.1); } } - self.ruleset_timing_roles.swap_remove(&ruleset); outcome?; let Some(mutex) = Arc::into_inner(results) else { @@ -3853,16 +3782,12 @@ mod tests { .parse_and_run_program(None, "(datatype Math (Num i64)) (let value (Num 1))") .unwrap(); - assert!( - egraph.phase_timings.leaves[phase_timers::FRONTEND_PARSE] > std::time::Duration::ZERO - ); - assert!(egraph.phase_timings.leaves[phase_timers::TYPECHECK] > std::time::Duration::ZERO); - assert!( - egraph.phase_timings.leaves[phase_timers::FRONTEND_OTHER] > std::time::Duration::ZERO - ); + assert!(egraph.process_timings.frontend_parse > std::time::Duration::ZERO); + assert!(egraph.process_timings.typecheck > std::time::Duration::ZERO); + assert!(egraph.process_timings.frontend_other > std::time::Duration::ZERO); let source_checker = egraph.proof_state.original_typechecking.as_ref().unwrap(); assert_eq!( - source_checker.phase_timings.leaves[phase_timers::TYPECHECK], + source_checker.process_timings.typecheck, std::time::Duration::ZERO, "the child checker must not retain time omitted from the outer summary" ); diff --git a/egglog/src/phase_timers.rs b/egglog/src/phase_timers.rs deleted file mode 100644 index b1d43764..00000000 --- a/egglog/src/phase_timers.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Exclusive wall-clock accounting outside ruleset execution. -//! -//! Static slices make paths the recording structure without allocating at -//! timer sites. The persisted summary converts them to owned path segments. - -use std::time::Duration; - -use indexmap::IndexMap; - -pub(crate) type TimingPath = &'static [&'static str]; - -pub(crate) const TYPECHECK: TimingPath = &["typecheck", "total"]; -pub(crate) const FRONTEND_PARSE: TimingPath = &["frontend", "parse"]; -pub(crate) const FRONTEND_OTHER: TimingPath = &["frontend", "other"]; -pub(crate) const FRONTEND_INSTALL: TimingPath = &["frontend", "install"]; -pub(crate) const COMMANDS_ACTIONS: TimingPath = &["commands", "actions"]; -pub(crate) const COMMANDS_CHECK: TimingPath = &["commands", "check"]; -pub(crate) const COMMANDS_OTHER: TimingPath = &["commands", "other"]; - -const STABLE_PROCESS_PATHS: [TimingPath; 7] = [ - TYPECHECK, - FRONTEND_PARSE, - FRONTEND_OTHER, - FRONTEND_INSTALL, - COMMANDS_ACTIONS, - COMMANDS_CHECK, - COMMANDS_OTHER, -]; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum RulesetTimingRole { - Program, - EqualityMaintenance, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct PhaseTimings { - pub(crate) leaves: IndexMap, -} - -impl Default for PhaseTimings { - fn default() -> Self { - Self { - leaves: STABLE_PROCESS_PATHS - .into_iter() - .map(|path| (path, Duration::ZERO)) - .collect(), - } - } -} - -impl PhaseTimings { - /// Accumulate one exclusive interval under exactly one path. - pub(crate) fn add(&mut self, path: TimingPath, duration: Duration) { - assert!(!path.is_empty(), "timing paths must not be empty"); - *self.leaves.entry(path).or_default() += duration; - } - - pub(crate) fn total(&self) -> Duration { - self.leaves.values().copied().sum() - } - - pub(crate) fn timing_leaves(&self) -> Vec<(Vec, Duration)> { - self.leaves - .iter() - .map(|(path, duration)| { - ( - path.iter().map(|segment| (*segment).to_owned()).collect(), - *duration, - ) - }) - .collect() - } -} diff --git a/egglog/src/scheduler.rs b/egglog/src/scheduler.rs index 03283506..f3a6e55c 100644 --- a/egglog/src/scheduler.rs +++ b/egglog/src/scheduler.rs @@ -201,13 +201,13 @@ impl EGraph { let Some(r) = rulesets.get(ruleset) else { return Err(Error::BackendError(format!("no such ruleset: {ruleset}"))); }; - match r { - Ruleset::Rules(rules) => { + match &r.kind { + RulesetKind::Rules(rules) => { for (rule_name, (core_rule, _)) in rules.iter() { ids.push((rule_name.clone(), core_rule)); } } - Ruleset::Combined(sub_rulesets) => { + RulesetKind::Combined(sub_rulesets) => { for sub_ruleset in sub_rulesets { collect_rules(sub_ruleset, rulesets, ids)?; } @@ -225,6 +225,7 @@ impl EGraph { self.rulesets = rulesets; return Err(e); } + let timing_role = rulesets[ruleset].timing_role; let mut schedulers = std::mem::take(&mut self.schedulers); let result = (|| -> Result { // Step 1: build all the query/action rules and worklist if have not already @@ -288,8 +289,8 @@ impl EGraph { .map_err(|e| Error::BackendError(e.to_string()))?; // Step 5: combine the reports - let mut query_report = RunReport::singleton(ruleset, query_iter_report); - let mut action_report = RunReport::singleton(ruleset, action_iter_report); + let mut query_report = RunReport::singleton(ruleset, timing_role, query_iter_report); + let mut action_report = RunReport::singleton(ruleset, timing_role, action_iter_report); // query matches don't count query_report.updated = false; @@ -309,7 +310,6 @@ impl EGraph { self.schedulers = schedulers; if let Ok(report) = &result { - self.record_ruleset_timing_role(ruleset); self.overall_run_report.union(report.clone()); } result @@ -510,13 +510,18 @@ mod test { // Because of semi-naive, the exact rules that are run are more than just `test-rule` assert!( report - .search_and_apply_time_per_rule + .search_and_apply_time_per_rule() .keys() .all(|k| k.starts_with("test-rule")) ); assert_eq!( - report.ruleset_timings.keys().collect::>(), - [&"test".into()] + report + .timings() + .rulesets + .iter() + .map(|timing| timing.name.as_ref()) + .collect::>(), + ["test"] ); if report.can_stop { diff --git a/egglog/tests/integration_test.rs b/egglog/tests/integration_test.rs index bf3eb661..878b04b5 100644 --- a/egglog/tests/integration_test.rs +++ b/egglog/tests/integration_test.rs @@ -917,7 +917,7 @@ fn test_print_stats() { let outputs = EGraph::default().parse_and_run_program(None, s).unwrap(); assert_eq!( outputs[1].to_string(), - "Overall statistics:\nRuleset : assembly 0.000s, search 0.000s, apply 0.000s, unattributed 0.000s, merge 0.000s, rebuild 0.000s\n" + "Overall statistics:\nRuleset : assembly 0.000s, search 0.000s, apply 0.000s, unattributed 0.000s, merge 0.000s\nNative rebuild: 0.000s\n" ); } diff --git a/egglog/tests/timing_summary_cli.rs b/egglog/tests/timing_summary_cli.rs index 8b27d935..3de513ea 100644 --- a/egglog/tests/timing_summary_cli.rs +++ b/egglog/tests/timing_summary_cli.rs @@ -30,22 +30,13 @@ fn assert_duration(value: &serde_json::Value) { assert!(duration["nanos"].is_u64()); } -fn timing_leaf(summary: &serde_json::Value, path: &[&str]) -> u64 { - summary["timings"] +fn ruleset<'a>(summary: &'a serde_json::Value, name: &str) -> &'a serde_json::Value { + summary["rulesets"] .as_array() .unwrap() .iter() - .find(|leaf| { - leaf["path"] - .as_array() - .unwrap() - .iter() - .map(|segment| segment.as_str().unwrap()) - .eq(path.iter().copied()) - }) - .unwrap_or_else(|| panic!("missing timing leaf {path:?}"))["ns"] - .as_u64() - .unwrap() + .find(|ruleset| ruleset["name"] == name) + .unwrap_or_else(|| panic!("missing ruleset {name:?}")) } #[test] @@ -77,15 +68,18 @@ fn checks_have_the_same_command_timing_path_with_and_without_term_encoding() { let summary: serde_json::Value = serde_json::from_slice(&std::fs::read(&summary_path).unwrap()).unwrap(); - assert!(timing_leaf(&summary, &["commands", "check"]) > 0); - assert!(!summary["timings"].as_array().unwrap().iter().any(|leaf| { - let path = leaf["path"].as_array().unwrap(); - path.first().and_then(serde_json::Value::as_str) == Some("program") - && path - .last() - .and_then(serde_json::Value::as_str) - .is_some_and(|name| name.contains("check_facts_ruleset")) - })); + assert!(summary["commands_check_ns"].as_u64().unwrap() > 0); + assert!( + !summary["rulesets"] + .as_array() + .unwrap() + .iter() + .any(|ruleset| { + ruleset["name"] + .as_str() + .is_some_and(|name| name.contains("check_facts_ruleset")) + }) + ); std::fs::remove_dir_all(directory).unwrap(); } @@ -121,26 +115,25 @@ fn encoded_equality_rulesets_are_tagged_by_role_not_mixed_with_program_rules() { ); let summary: serde_json::Value = serde_json::from_slice(&std::fs::read(&summary_path).unwrap()).unwrap(); - let maintenance_names = summary["timings"] + let maintenance_names = summary["rulesets"] .as_array() .unwrap() .iter() - .filter_map(|leaf| { - let path = leaf["path"].as_array().unwrap(); - (path.len() == 3 - && path[0].as_str() == Some("equality") - && path[1].as_str() != Some("rebuild")) - .then(|| path[2].as_str().unwrap().to_owned()) - }) + .filter(|ruleset| ruleset["role"] == "equality") + .map(|ruleset| ruleset["name"].as_str().unwrap().to_owned()) .collect::>(); assert!(!maintenance_names.is_empty()); - assert!(!summary["timings"].as_array().unwrap().iter().any(|leaf| { - let path = leaf["path"].as_array().unwrap(); - path.len() == 3 - && path[0].as_str() == Some("program") - && maintenance_names.contains(path[2].as_str().unwrap()) - })); + assert!( + !summary["rulesets"] + .as_array() + .unwrap() + .iter() + .any(|ruleset| { + ruleset["role"] == "program" + && maintenance_names.contains(ruleset["name"].as_str().unwrap()) + }) + ); std::fs::remove_dir_all(directory).unwrap(); } @@ -186,43 +179,46 @@ fn timing_summary_is_compact_and_works_with_every_report_level() { assert!(!bytes[..bytes.len() - 1].contains(&b'\n')); let summary: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(summary.as_object().unwrap().len(), 2); - assert_eq!(summary["schema_version"], 3); - let timings = summary["timings"].as_array().unwrap(); - assert_eq!(timings.len(), 19); - let paths = timings - .iter() - .map(|leaf| { - leaf["path"] - .as_array() - .unwrap() - .iter() - .map(|segment| segment.as_str().unwrap().to_owned()) - .collect::>() - }) - .collect::>(); - assert!(paths.windows(2).all(|pair| pair[0] < pair[1])); - assert_eq!(timing_leaf(&summary, &["commands", "check"]), 0); - for path in [ - &["frontend", "parse"][..], - &["frontend", "other"], - &["frontend", "install"], - &["typecheck", "total"], - &["commands", "actions"], - &["commands", "other"], + assert_eq!(summary.as_object().unwrap().len(), 10); + assert_eq!(summary["schema_version"], 4); + assert_eq!(summary["commands_check_ns"], 0); + for field in [ + "frontend_parse_ns", + "frontend_other_ns", + "frontend_install_ns", + "typecheck_ns", + "commands_actions_ns", + "commands_other_ns", ] { - assert!(timing_leaf(&summary, path) > 0, "expected nonzero {path:?}"); + assert!( + summary[field].as_u64().unwrap() > 0, + "expected nonzero {field}" + ); } - for ruleset in ["alpha", "zeta"] { - for phase in ["assembly", "search", "apply", "execution", "merge"] { - timing_leaf(&summary, &["program", phase, ruleset]); + let rulesets = summary["rulesets"].as_array().unwrap(); + assert_eq!(rulesets.len(), 2); + assert_eq!(rulesets[0]["name"], "alpha"); + assert_eq!(rulesets[1]["name"], "zeta"); + for ruleset_name in ["alpha", "zeta"] { + let timing = ruleset(&summary, ruleset_name); + assert_eq!(timing["role"], "program"); + for phase in [ + "assembly_ns", + "search_ns", + "apply_ns", + "execution_ns", + "merge_ns", + ] { + assert!(timing[phase].is_u64()); } - timing_leaf(&summary, &["equality", "rebuild", ruleset]); } + assert!(summary["native_rebuild_ns"].is_u64()); let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&report_path).unwrap()).unwrap(); for iteration in report["iterations"].as_array().unwrap() { - let split = iteration["rule_set_report"]["pre_merge"]["Split"] + assert!(iteration["name"].is_string()); + assert_eq!(iteration["role"], "program"); + let split = iteration["report"]["rule_set_report"]["pre_merge"]["Split"] .as_object() .unwrap(); assert_eq!(split.len(), 3); @@ -271,7 +267,7 @@ fn parallel_saved_report_uses_combined_pre_merge_shape() { let iterations = report["iterations"].as_array().unwrap(); assert!(!iterations.is_empty()); for iteration in iterations { - let combined = iteration["rule_set_report"]["pre_merge"]["Combined"] + let combined = iteration["report"]["rule_set_report"]["pre_merge"]["Combined"] .as_object() .unwrap(); assert_eq!(combined.len(), 1); @@ -326,8 +322,8 @@ fn stdin_program_writes_timing_summary() { let bytes = std::fs::read(&summary_path).unwrap(); assert_eq!(bytes.last(), Some(&b'\n')); let summary: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(summary["schema_version"], 3); - assert!(summary["timings"].is_array()); + assert_eq!(summary["schema_version"], 4); + assert!(summary["rulesets"].is_array()); std::fs::remove_dir_all(directory).unwrap(); } diff --git a/tests/report_fixtures.py b/tests/report_fixtures.py index 94135c79..38a81794 100644 --- a/tests/report_fixtures.py +++ b/tests/report_fixtures.py @@ -3,14 +3,14 @@ from __future__ import annotations from pathlib import Path -from typing import Literal from benchmarking import models from benchmarking.reports.store import ( REPORT_SCHEMA_VERSION, ReportRecord, ReportStore, - TimingLeafRecord, + RulesetTimingRecord, + RulesetTimingRole, TimingSummaryRecord, ) @@ -68,24 +68,23 @@ def make_ruleset_timing( apply_ns: int = 200_000_000, execution_ns: int = 0, merge_ns: int = 200_000_000, - rebuild_ns: int = 100_000_000, - role: Literal["program", "equality"] = "program", -) -> tuple[TimingLeafRecord, ...]: + role: RulesetTimingRole = "program", +) -> RulesetTimingRecord: """Construct one valid ruleset timing fixture.""" - responsibility = "equality" if role == "equality" else "program" - return ( - {"path": [responsibility, "assembly", name], "ns": assembly_ns}, - {"path": [responsibility, "search", name], "ns": search_ns}, - {"path": [responsibility, "apply", name], "ns": apply_ns}, - {"path": [responsibility, "execution", name], "ns": execution_ns}, - {"path": [responsibility, "merge", name], "ns": merge_ns}, - {"path": ["equality", "rebuild", name], "ns": rebuild_ns}, - ) + return { + "name": name, + "role": role, + "assembly_ns": assembly_ns, + "search_ns": search_ns, + "apply_ns": apply_ns, + "execution_ns": execution_ns, + "merge_ns": merge_ns, + } def make_timing_summary( - *rulesets: tuple[TimingLeafRecord, ...], + *rulesets: RulesetTimingRecord, typecheck_ns: int = 0, frontend_parse_ns: int = 0, frontend_other_ns: int = 0, @@ -93,24 +92,21 @@ def make_timing_summary( commands_actions_ns: int = 0, commands_check_ns: int = 0, commands_other_ns: int = 0, + native_rebuild_ns: int = 100_000_000, ) -> TimingSummaryRecord: - """Construct a valid V3 timing-summary fixture.""" - - timing_groups = rulesets or (make_ruleset_timing(),) - timings: list[TimingLeafRecord] = [ - {"path": ["typecheck", "total"], "ns": typecheck_ns}, - {"path": ["frontend", "parse"], "ns": frontend_parse_ns}, - {"path": ["frontend", "other"], "ns": frontend_other_ns}, - {"path": ["frontend", "install"], "ns": frontend_install_ns}, - {"path": ["commands", "actions"], "ns": commands_actions_ns}, - {"path": ["commands", "check"], "ns": commands_check_ns}, - {"path": ["commands", "other"], "ns": commands_other_ns}, - ] - timings.extend(leaf for group in timing_groups for leaf in group) - timings.sort(key=lambda leaf: leaf["path"]) + """Construct a valid dense timing-summary fixture.""" + return { - "schema_version": 3, - "timings": timings, + "schema_version": 4, + "typecheck_ns": typecheck_ns, + "frontend_parse_ns": frontend_parse_ns, + "frontend_other_ns": frontend_other_ns, + "frontend_install_ns": frontend_install_ns, + "commands_actions_ns": commands_actions_ns, + "commands_check_ns": commands_check_ns, + "commands_other_ns": commands_other_ns, + "native_rebuild_ns": native_rebuild_ns, + "rulesets": list(rulesets or (make_ruleset_timing(),)), } diff --git a/tests/test_collection.py b/tests/test_collection.py index b0b26eda..f2c9abd7 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -583,12 +583,25 @@ def fake_run_command(command: list[str], checkout_path: Path, timeout_sec: int) summary_path.write_text( json.dumps( { - "schema_version": 3, - "timings": [ - {"path": ["program", "search", "rules"], "ns": 4}, - {"path": ["program", "apply", "rules"], "ns": 6}, - {"path": ["program", "execution", "rules"], "ns": 10}, - {"path": ["program", "merge", "rules"], "ns": 20}, + "schema_version": 4, + "typecheck_ns": 1, + "frontend_parse_ns": 2, + "frontend_other_ns": 3, + "frontend_install_ns": 4, + "commands_actions_ns": 5, + "commands_check_ns": 6, + "commands_other_ns": 7, + "native_rebuild_ns": 8, + "rulesets": [ + { + "name": "rules", + "role": "program", + "assembly_ns": 3, + "search_ns": 4, + "apply_ns": 6, + "execution_ns": 10, + "merge_ns": 20, + } ], } ), @@ -604,12 +617,18 @@ def fake_run_command(command: list[str], checkout_path: Path, timeout_sec: int) assert "--proofs" not in commands[0] assert "--proofs" in commands[1] assert off.timing_summary is not None - assert off.timing_summary["timings"] == [ - {"path": ["program", "search", "rules"], "ns": 4}, - {"path": ["program", "apply", "rules"], "ns": 6}, - {"path": ["program", "execution", "rules"], "ns": 10}, - {"path": ["program", "merge", "rules"], "ns": 20}, + assert off.timing_summary["rulesets"] == [ + { + "name": "rules", + "role": "program", + "assembly_ns": 3, + "search_ns": 4, + "apply_ns": 6, + "execution_ns": 10, + "merge_ns": 20, + } ] + assert off.timing_summary["native_rebuild_ns"] == 8 assert proofs.timing_summary is not None diff --git a/tests/test_report_analysis.py b/tests/test_report_analysis.py index 7e91e522..672272dd 100644 --- a/tests/test_report_analysis.py +++ b/tests/test_report_analysis.py @@ -10,7 +10,7 @@ from benchmarking import models from benchmarking.reports.analysis import analyze_pair -from benchmarking.reports.store import ReportRecord, ReportStore, TimingSummaryRecord +from benchmarking.reports.store import ReportRecord, ReportStore from .report_fixtures import make_record, make_ruleset_timing, make_target, make_timing_summary, write_report @@ -68,10 +68,10 @@ def test_analysis_computes_only_the_requested_detail_rows(tmp_path: Path) -> Non rulesets = analyze_pair(store, comparison, "rulesets") assert len(summary.summary) == 5 - assert not summary.files and not summary.decomposition and not summary.rulesets - assert files.files and not files.decomposition and not files.rulesets - assert phases.files and phases.decomposition and not phases.rulesets - assert rulesets.files and rulesets.decomposition and rulesets.rulesets + assert not summary.files and not summary.timing + assert files.files and not files.timing + assert phases.files and phases.timing + assert rulesets.files and rulesets.timing def test_pair_statistics_and_fieller_intervals(tmp_path: Path) -> None: @@ -264,8 +264,8 @@ def test_mechanism_buckets_are_additive_and_residual_closes_to_wall(tmp_path: Pa apply_ns=200, execution_ns=17, merge_ns=300, - rebuild_ns=400, - ) + ), + native_rebuild_ns=400, ) candidate_timing = make_timing_summary( make_ruleset_timing( @@ -273,8 +273,8 @@ def test_mechanism_buckets_are_additive_and_residual_closes_to_wall(tmp_path: Pa apply_ns=100, execution_ns=23, merge_ns=600, - rebuild_ns=200, - ) + ), + native_rebuild_ns=200, ) write_report( report, @@ -294,20 +294,18 @@ def test_mechanism_buckets_are_additive_and_residual_closes_to_wall(tmp_path: Pa ), ) - suite, file_row = analyze_pair(ReportStore(report), comparison, "phases").decomposition + suite, file_row = analyze_pair(ReportStore(report), comparison, "phases").timing assert suite.file_order is None assert file_row.file_order == 0 assert file_row.wall_delta_ns == pytest.approx(500.0) - assert [cell.delta_ns for cell in file_row.mechanisms] == pytest.approx([0.0, 0.0, 306.0, -200.0, 0.0, 394.0]) - assert [cell.slowdown_share for cell in file_row.mechanisms] == pytest.approx([0.0, 0.0, 0.612, -0.4, 0.0, 0.788]) - assert sum(cell.delta_ns or 0.0 for cell in file_row.mechanisms) == pytest.approx(file_row.wall_delta_ns) - assert sum(cell.slowdown_share or 0.0 for cell in file_row.mechanisms) == pytest.approx(1.0) + assert file_row.mechanism_deltas == pytest.approx([0.0, 0.0, 306.0, -200.0, 0.0, 394.0]) + assert sum(delta or 0.0 for delta in file_row.mechanism_deltas) == pytest.approx(file_row.wall_delta_ns) assert suite.wall_delta_ns == file_row.wall_delta_ns - assert suite.mechanisms == file_row.mechanisms + assert suite.mechanism_deltas == file_row.mechanism_deltas -def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp_path: Path) -> None: +def test_process_rulesets_and_global_rebuild_are_each_subtracted_from_residual(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path) timing = make_timing_summary( @@ -317,7 +315,6 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp apply_ns=41, execution_ns=43, merge_ns=47, - rebuild_ns=53, ), frontend_parse_ns=11, typecheck_ns=13, @@ -326,6 +323,7 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp commands_actions_ns=23, commands_check_ns=7, commands_other_ns=29, + native_rebuild_ns=53, ) zero_timing = make_timing_summary( make_ruleset_timing( @@ -334,8 +332,8 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp apply_ns=0, execution_ns=0, merge_ns=0, - rebuild_ns=0, - ) + ), + native_rebuild_ns=0, ) write_report( report, @@ -356,49 +354,14 @@ def test_nested_process_and_ruleset_leaves_are_each_subtracted_from_residual(tmp ) views = analyze_pair(ReportStore(report), comparison, "rulesets") - file_row = views.decomposition[1] + file_row = views.timing[1] assert file_row.wall_delta_ns == pytest.approx(500.0) - assert [cell.delta_ns for cell in file_row.mechanisms] == pytest.approx([13.0, 47.0, 199.0, 53.0, 59.0, 129.0]) - assert sum(cell.delta_ns or 0.0 for cell in file_row.mechanisms) == pytest.approx(500.0) - program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") - equality = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "equality") - native_rebuild = next(row for row in views.rulesets if row.kind == "native_rebuild") - assert program.delta.phases == pytest.approx((31, 37, 41, 43, 47, 0)) - assert program.delta.total == file_row.mechanisms.program.delta_ns == 199 - assert equality.delta.phases == pytest.approx((0, 0, 0, 0, 0, 53)) - assert equality.delta.total == file_row.mechanisms.equality.delta_ns == 53 - assert native_rebuild.delta == equality.delta - - -@pytest.mark.parametrize( - ("path", "message"), - ((["residual", "stored"], "residual is derived"), (["mystery", "work"], "unknown timing responsibility")), -) -def test_invalid_timing_responsibilities_are_rejected_by_the_reader( - tmp_path: Path, - path: list[str], - message: str, -) -> None: - report = tmp_path / "report.jsonl" - comparison = _comparison(tmp_path) - invalid = cast( - TimingSummaryRecord, - {"schema_version": 3, "timings": [{"path": path, "ns": 1}]}, - ) - write_report( - report, - make_record(0, started_at="2026-07-15T12:00:00Z", binary_sha256="sha256:baseline"), - make_record( - 1, - started_at="2026-07-15T12:00:01Z", - binary_sha256="sha256:candidate", - timing_summary=invalid, - ), - ) - - with pytest.raises(ValueError, match=message): - analyze_pair(ReportStore(report), comparison, "phases") + assert file_row.mechanism_deltas == pytest.approx([13.0, 47.0, 199.0, 53.0, 59.0, 129.0]) + assert sum(delta or 0.0 for delta in file_row.mechanism_deltas) == pytest.approx(500.0) + assert file_row.program.phases == pytest.approx((31, 37, 41, 43, 47, 0)) + assert file_row.equality.phases == pytest.approx((0, 0, 0, 0, 0, 53)) + assert file_row.equality.native_rebuild_delta_ns == 53 def test_mechanism_decomposition_uses_endpoint_means_and_wall_context(tmp_path: Path) -> None: @@ -417,19 +380,18 @@ def test_mechanism_decomposition_uses_endpoint_means_and_wall_context(tmp_path: binary_sha256=binary_sha256, wall_sec=wall_ns / 1_000_000_000.0, timing_summary=make_timing_summary( - make_ruleset_timing(search_ns=search_ns, apply_ns=0, merge_ns=0, rebuild_ns=0) + make_ruleset_timing(search_ns=search_ns, apply_ns=0, merge_ns=0), + native_rebuild_ns=0, ), ) ) write_report(report, *records) - file_row = analyze_pair(ReportStore(report), comparison, "phases").decomposition[1] + file_row = analyze_pair(ReportStore(report), comparison, "phases").timing[1] assert file_row.wall_delta_ns == pytest.approx(500.0) - assert file_row.mechanisms.program.delta_ns == 100 - assert file_row.mechanisms.program.slowdown_share == pytest.approx(0.2) - assert file_row.mechanisms.residual.delta_ns == 400 - assert file_row.mechanisms.residual.slowdown_share == pytest.approx(0.8) + assert file_row.program.phases.total == 100 + assert file_row.residual_delta_ns == 400 def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_path: Path) -> None: @@ -441,7 +403,6 @@ def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_pa apply_ns=0, execution_ns=0, merge_ns=0, - rebuild_ns=0, ) write_report( report, @@ -450,9 +411,10 @@ def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_pa started_at="2026-07-15T12:00:00Z", binary_sha256="sha256:baseline", timing_summary=make_timing_summary( - make_ruleset_timing("baseline-only", search_ns=10, apply_ns=0, merge_ns=0, rebuild_ns=0), - make_ruleset_timing("sporadic", search_ns=8, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing("baseline-only", search_ns=10, apply_ns=0, merge_ns=0), + make_ruleset_timing("sporadic", search_ns=8, apply_ns=0, merge_ns=0), zero, + native_rebuild_ns=0, ), ), make_record( @@ -460,8 +422,9 @@ def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_pa started_at="2026-07-15T12:00:01Z", binary_sha256="sha256:baseline", timing_summary=make_timing_summary( - make_ruleset_timing("baseline-only", search_ns=10, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing("baseline-only", search_ns=10, apply_ns=0, merge_ns=0), zero, + native_rebuild_ns=0, ), ), make_record( @@ -469,16 +432,16 @@ def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_pa started_at="2026-07-15T12:00:02Z", binary_sha256="sha256:candidate", timing_summary=make_timing_summary( - make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0), make_ruleset_timing( "assembly-only", assembly_ns=5, search_ns=0, apply_ns=0, merge_ns=0, - rebuild_ns=0, ), zero, + native_rebuild_ns=0, ), ), make_record( @@ -486,56 +449,44 @@ def test_ruleset_union_aligns_absence_with_zero_and_aggregates_iterations(tmp_pa started_at="2026-07-15T12:00:03Z", binary_sha256="sha256:candidate", timing_summary=make_timing_summary( - make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0, rebuild_ns=0), + make_ruleset_timing("candidate-only", search_ns=20, apply_ns=0, merge_ns=0), make_ruleset_timing( "assembly-only", assembly_ns=5, search_ns=0, apply_ns=0, merge_ns=0, - rebuild_ns=0, ), zero, + native_rebuild_ns=0, ), ), ) views = analyze_pair(ReportStore(report), comparison, "rulesets") - rows = {row.name: row for row in views.rulesets if row.kind == "ruleset"} - - assert rows["baseline-only"].delta.phases.search == -10 - assert rows["candidate-only"].delta.phases.search == 20 - assert rows["sporadic"].delta.phases.search == -4 - assert rows["assembly-only"].delta.phases.assembly == 5 - assert rows["assembly-only"].delta.total == 5 + file_row = views.timing[1] + rows = {row.name: row for row in file_row.program.rulesets} + + assert rows["baseline-only"].phases.search == -10 + assert rows["candidate-only"].phases.search == 20 + assert rows["sporadic"].phases.search == -4 + assert rows["assembly-only"].phases.assembly == 5 + assert rows["assembly-only"].phases.total == 5 assert "recorded-zero" not in rows - program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") - assert program.ruleset_count == 4 - assert program.delta.total == 11 + assert len(file_row.program.rulesets) == 4 + assert file_row.program.phases.total == 11 -def test_ruleset_drilldown_separates_program_work_from_native_rebuild(tmp_path: Path) -> None: +def test_role_changes_are_separate_ruleset_changes_and_rebuild_is_global(tmp_path: Path) -> None: report = tmp_path / "report.jsonl" comparison = _comparison(tmp_path) - baseline = cast( - TimingSummaryRecord, - { - "schema_version": 3, - "timings": [ - {"path": ["program", "search", "rules/λ"], "ns": 0}, - {"path": ["equality", "rebuild", "rules/λ"], "ns": 0}, - ], - }, + baseline = make_timing_summary( + make_ruleset_timing("rules/λ", search_ns=10, apply_ns=0, merge_ns=0), + native_rebuild_ns=7, ) - candidate = cast( - TimingSummaryRecord, - { - "schema_version": 3, - "timings": [ - {"path": ["program", "search", "rules/λ"], "ns": 10}, - {"path": ["equality", "rebuild", "rules/λ"], "ns": 7}, - ], - }, + candidate = make_timing_summary( + make_ruleset_timing("rules/λ", role="equality", search_ns=12, apply_ns=0, merge_ns=0), + native_rebuild_ns=3, ) write_report( report, @@ -553,18 +504,12 @@ def test_ruleset_drilldown_separates_program_work_from_native_rebuild(tmp_path: ), ) - rows = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets - program_rule = next(row for row in rows if row.kind == "ruleset") - native_rebuild = next(row for row in rows if row.kind == "native_rebuild") - - assert program_rule.name == "rules/λ" - assert program_rule.mechanism == "program" - assert program_rule.delta.phases.search == 10 - assert program_rule.delta.phases.rebuild == 0 - assert program_rule.delta.total == 10 - assert native_rebuild.mechanism == "equality" - assert native_rebuild.delta.phases.rebuild == 7 - assert native_rebuild.delta.total == 7 + file_row = analyze_pair(ReportStore(report), comparison, "rulesets").timing[1] + assert file_row.program.rulesets[0].phases.search == -10 + assert file_row.equality.rulesets[0].phases.search == 12 + assert file_row.equality.native_rebuild_delta_ns == -4 + assert file_row.program.rulesets[0].phases.rebuild == 0 + assert file_row.equality.rulesets[0].phases.rebuild == 0 def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: Path) -> None: @@ -578,7 +523,6 @@ def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: P apply_ns=5, execution_ns=7, merge_ns=11, - rebuild_ns=13, ), make_ruleset_timing( "maintenance", @@ -587,9 +531,9 @@ def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: P apply_ns=23, execution_ns=29, merge_ns=31, - rebuild_ns=37, role="equality", ), + native_rebuild_ns=50, ) write_report( report, @@ -597,7 +541,10 @@ def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: P 0, started_at="2026-07-15T12:00:00Z", binary_sha256="sha256:baseline", - timing_summary=cast(TimingSummaryRecord, {"schema_version": 3, "timings": []}), + timing_summary=make_timing_summary( + make_ruleset_timing(search_ns=0, apply_ns=0, merge_ns=0), + native_rebuild_ns=0, + ), ), make_record( 1, @@ -608,74 +555,13 @@ def test_ruleset_parent_groups_equal_program_and_equality_mechanisms(tmp_path: P ) views = analyze_pair(ReportStore(report), comparison, "rulesets") - program = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "program") - equality = next(row for row in views.rulesets if row.kind == "aggregate" and row.mechanism == "equality") - maintenance = next( - row - for row in views.rulesets - if row.kind == "ruleset" and row.mechanism == "equality" and row.name == "maintenance" - ) - native_rebuild = next(row for row in views.rulesets if row.kind == "native_rebuild") - mechanisms = views.decomposition[1].mechanisms - - assert program.delta.total == mechanisms.program.delta_ns == 28 - assert maintenance.delta.total == 156 - assert native_rebuild.delta.total == 13 - assert equality.delta.total == mechanisms.equality.delta_ns == 169 - assert maintenance.delta.total + native_rebuild.delta.total == equality.delta.total - for parent in (program, equality): - children = [row for row in views.rulesets if row.kind != "aggregate" and row.mechanism == parent.mechanism] - assert sum(row.delta.total for row in children) == parent.delta.total - assert all(sum(row.delta.phases[index] for row in children) == parent.delta.phases[index] for index in range(6)) - - -def test_program_children_are_fixed_top_five_plus_exact_per_group_other(tmp_path: Path) -> None: - report = tmp_path / "report.jsonl" - comparison = _comparison(tmp_path) - names = tuple(reversed(tuple(f"rules-{index:02d}" for index in range(12)))) - baseline_rules = tuple( - make_ruleset_timing(name, search_ns=100, apply_ns=0, merge_ns=0, rebuild_ns=0) for name in names - ) - candidate_rules = tuple( - make_ruleset_timing(name, search_ns=101, apply_ns=0, merge_ns=0, rebuild_ns=0) for name in names - ) - write_report( - report, - make_record( - 0, - started_at="2026-07-15T12:00:00Z", - binary_sha256="sha256:baseline", - timing_summary=make_timing_summary(*baseline_rules), - ), - make_record( - 1, - started_at="2026-07-15T12:00:01Z", - binary_sha256="sha256:candidate", - timing_summary=make_timing_summary(*candidate_rules), - ), - ) - - rulesets = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets - - parents = [row for row in rulesets if row.kind == "aggregate"] - contributors = [row for row in rulesets if row.kind == "ruleset"] - other = next(row for row in rulesets if row.kind == "other") - assert [(row.mechanism, row.ruleset_count, row.delta.total) for row in parents] == [ - ("program", 12, 12), - ("equality", 0, 0), - ] - assert [row.name for row in contributors] == [f"rules-{index:02d}" for index in range(5)] - assert all(row.mechanism == "program" for row in contributors) - assert other.ruleset_count == 7 - assert other.delta.total == 7 - assert other.delta.phases.search == 7 - program_parent = parents[0] - assert sum(row.delta.total for row in contributors) + other.delta.total == program_parent.delta.total - assert all( - sum(row.delta.phases[index] for row in contributors) + other.delta.phases[index] - == program_parent.delta.phases[index] - for index in range(6) - ) + file_row = views.timing[1] + maintenance = file_row.equality.rulesets[0] + assert file_row.program.phases.total == file_row.mechanism_deltas[2] == 28 + assert maintenance.phases.total == 119 + assert file_row.equality.native_rebuild_delta_ns == 50 + assert file_row.equality.phases.total == file_row.mechanism_deltas[3] == 169 + assert maintenance.phases.total + file_row.equality.native_rebuild_delta_ns == file_row.equality.phases.total def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tmp_path: Path) -> None: @@ -689,7 +575,6 @@ def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tm apply_ns=0, execution_ns=0, merge_ns=0, - rebuild_ns=13, ) baseline_maintenance = tuple( make_ruleset_timing( @@ -699,7 +584,6 @@ def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tm apply_ns=0, execution_ns=0, merge_ns=0, - rebuild_ns=0, role="equality", ) for name in names @@ -712,7 +596,6 @@ def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tm apply_ns=0, execution_ns=0, merge_ns=0, - rebuild_ns=0, role="equality", ) for index, name in enumerate(names) @@ -724,24 +607,52 @@ def test_all_maintenance_children_are_shown_and_zero_native_rebuild_is_hidden(tm started_at="2026-07-15T12:00:00Z", binary_sha256="sha256:baseline", timing_summary=make_timing_summary(source, *baseline_maintenance), + wall_sec=0.000001, ), make_record( 1, started_at="2026-07-15T12:00:01Z", binary_sha256="sha256:candidate", timing_summary=make_timing_summary(source, *candidate_maintenance), + wall_sec=0.000001, ), ) - rulesets = analyze_pair(ReportStore(report), comparison, "rulesets").rulesets - maintenance = [row for row in rulesets if row.kind == "ruleset" and row.mechanism == "equality"] - equality = next(row for row in rulesets if row.kind == "aggregate" and row.mechanism == "equality") + file_row = analyze_pair(ReportStore(report), comparison, "rulesets").timing[1] + assert len(file_row.equality.rulesets) == 7 + assert [row.name for row in file_row.equality.rulesets] == list(names) + assert file_row.equality.phases.total == sum(range(1, 8)) + assert file_row.equality.native_rebuild_delta_ns == 0 + + +def test_negative_residual_is_preserved_as_an_attribution_warning(tmp_path: Path) -> None: + report = tmp_path / "report.jsonl" + comparison = _comparison(tmp_path) + timing = make_timing_summary( + make_ruleset_timing(search_ns=10, apply_ns=0, merge_ns=0), + native_rebuild_ns=0, + ) + write_report( + report, + make_record( + 0, + started_at="2026-07-15T12:00:00Z", + binary_sha256="sha256:baseline", + wall_sec=5 / 1_000_000_000, + timing_summary=timing, + ), + make_record( + 1, + started_at="2026-07-15T12:00:01Z", + binary_sha256="sha256:candidate", + wall_sec=6 / 1_000_000_000, + timing_summary=timing, + ), + ) - assert len(maintenance) == 7 - assert [row.name for row in maintenance] == list(reversed(names)) - assert equality.delta.total == sum(range(1, 8)) - assert not any(row.kind == "native_rebuild" for row in rulesets) - assert not any(row.kind == "other" and row.mechanism == "equality" for row in rulesets) + file_row = analyze_pair(ReportStore(report), comparison, "phases").timing[1] + assert file_row.residual_warning + assert file_row.residual_delta_ns == pytest.approx(1) def _fieller_bounds( diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index 35d465eb..87f347d7 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -13,7 +13,7 @@ from syrupy.assertion import SnapshotAssertion from benchmarking import models -from benchmarking.reports.analysis import PhaseValues, RulesetDelta +from benchmarking.reports.analysis import PhaseValues from benchmarking.reports.catalog import ReportCatalog, ReportMessage, ReportTable, report_id from benchmarking.reports.presentation import ( _important_phase_changes, @@ -363,8 +363,8 @@ def test_negative_residual_keeps_an_explicit_warning(tmp_path: Path) -> None: search_ns=1_200_000_000, apply_ns=0, merge_ns=0, - rebuild_ns=0, - ) + ), + native_rebuild_ns=0, ), ), make_record( @@ -378,8 +378,8 @@ def test_negative_residual_keeps_an_explicit_warning(tmp_path: Path) -> None: search_ns=1_100_000_000, apply_ns=0, merge_ns=0, - rebuild_ns=0, - ) + ), + native_rebuild_ns=0, ), ), ) @@ -392,9 +392,9 @@ def test_negative_residual_keeps_an_explicit_warning(tmp_path: Path) -> None: def test_important_phase_changes_use_the_documented_deterministic_threshold() -> None: - delta = RulesetDelta(20_000_000, PhaseValues(500_000, 10_000_000, 3_000_000, 2_000_000, 4_000_000, 500_000)) + phases = PhaseValues(500_000, 10_000_000, 3_000_000, 2_000_000, 4_000_000, 500_000) - assert _important_phase_changes(delta) == ( + assert _important_phase_changes(phases) == ( "◆ Search +10.0 ms; Apply +3.00 ms; Execution +2.00 ms; Merge +4.00 ms; …" ) @@ -488,8 +488,12 @@ def test_timed_out_file_has_missing_phase_cells_and_ruleset_status(tmp_path: Pat assert len(phase_table.rows) == 2 assert all(cell.display == "—" for row in phase_table.rows for cell in row.cells[1:]) ruleset_section = next(section for section in catalog.sections if section.id == "rulesets") - assert isinstance(ruleset_section.blocks[0], ReportMessage) - assert ruleset_section.blocks[0].text == "Status: timeout row selected" + status = next( + block + for block in ruleset_section.blocks + if isinstance(block, ReportMessage) and block.title == "Ruleset drivers — file.egg" + ) + assert status.text == "Status: timeout row selected" summary_section = next(section for section in catalog.sections if section.id == "summary") summary_table = next(block for block in summary_section.blocks if isinstance(block, ReportTable)) invalid = next(row for row in summary_table.rows if row.cells[4].raw == "invalid") @@ -538,15 +542,14 @@ def _pair_case(tmp_path: Path) -> tuple[Path, models.ComparisonSpec]: search_ns=int(wall * 300_000_000), apply_ns=int(wall * 120_000_000), merge_ns=80_000_000, - rebuild_ns=30_000_000, ), make_ruleset_timing( "finish", search_ns=int(wall * 100_000_000), apply_ns=int(wall * 60_000_000), merge_ns=20_000_000, - rebuild_ns=10_000_000, ), + native_rebuild_ns=40_000_000, ), ) ) @@ -584,7 +587,6 @@ def _six_file_pair_case(tmp_path: Path) -> tuple[Path, models.ComparisonSpec]: apply_ns=int((ruleset_order + 1) * (file_order + 1) * 1_000_000 * timing_factor), execution_ns=int((ruleset_order + 1) * 200_000 * timing_factor), merge_ns=int((ruleset_order + 1) * 500_000 * timing_factor), - rebuild_ns=int((ruleset_order + 1) * 250_000 * timing_factor), ) for ruleset_order in range(12) ) @@ -598,7 +600,12 @@ def _six_file_pair_case(tmp_path: Path) -> tuple[Path, models.ComparisonSpec]: target_label=endpoint.target.row.label, wall_sec=baseline_wall * wall_factor + round_index * 0.01, max_rss_bytes=(100 + file_order * 20 + endpoint_order * 10 + round_index) * 1_000_000, - timing_summary=make_timing_summary(*rulesets), + timing_summary=make_timing_summary( + *rulesets, + native_rebuild_ns=sum( + int((ruleset_order + 1) * 250_000 * timing_factor) for ruleset_order in range(12) + ), + ), ) ) write_report(report_path, *records) diff --git a/tests/test_report_store.py b/tests/test_report_store.py index 216f5247..4cb12d37 100644 --- a/tests/test_report_store.py +++ b/tests/test_report_store.py @@ -12,7 +12,7 @@ CacheKey, ReportRecord, ReportStore, - TimingLeafRecord, + RulesetTimingRecord, TimingSummaryRecord, parse_report_record, ) @@ -116,8 +116,8 @@ def test_typed_dict_schema_and_nested_values_round_trip(tmp_path: Path) -> None: apply_ns=5, execution_ns=4, merge_ns=7, - rebuild_ns=3, - ) + ), + native_rebuild_ns=3, ), ) @@ -128,9 +128,20 @@ def test_typed_dict_schema_and_nested_values_round_trip(tmp_path: Path) -> None: assert tuple(loaded) == tuple(ReportRecord.__annotations__) summary = cast(TimingSummaryRecord, loaded["timing_summary"]) assert tuple(summary) == tuple(TimingSummaryRecord.__annotations__) - timings = cast(list[TimingLeafRecord], summary["timings"]) - assert all(tuple(leaf) == tuple(TimingLeafRecord.__annotations__) for leaf in timings) - assert ["program", "search", "rules/λ"] in [leaf["path"] for leaf in timings] + rulesets = cast(list[RulesetTimingRecord], summary["rulesets"]) + assert all(tuple(ruleset) == tuple(RulesetTimingRecord.__annotations__) for ruleset in rulesets) + assert rulesets == [ + { + "name": "rules/λ", + "role": "program", + "assembly_ns": 0, + "search_ns": 6, + "apply_ns": 5, + "execution_ns": 4, + "merge_ns": 7, + } + ] + assert summary["native_rebuild_ns"] == 3 @pytest.mark.parametrize("schema_version", [None, 2], ids=["missing", "wrong"]) From 68d3f5f69b8af5d7b43010bf90920c77e52f170c Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 13:08:08 -0400 Subject: [PATCH 8/9] Consolidate cumulative timing reports --- egg-math-benchmark/src/main.rs | 53 ++- .../tests/scheduler_reporting.rs | 8 +- egglog/egglog-reports/src/lib.rs | 389 ++++++++---------- egglog/src/lib.rs | 75 ++-- egglog/src/scheduler.rs | 12 +- 5 files changed, 245 insertions(+), 292 deletions(-) diff --git a/egg-math-benchmark/src/main.rs b/egg-math-benchmark/src/main.rs index 946b821c..82995e8b 100644 --- a/egg-math-benchmark/src/main.rs +++ b/egg-math-benchmark/src/main.rs @@ -1,9 +1,7 @@ use anyhow::{Context, Result, ensure}; use clap::{Parser, ValueEnum}; use egg::{RecExpr, Runner, SimpleScheduler, StopReason}; -use egglog_reports::{ - PreMergeTiming, ProcessTimings, RulesetTiming, RulesetTimingRole, RunTimings, TimingSummary, -}; +use egglog_reports::{RulesetTimingRecord, RulesetTimingRole, TimingSummary}; use std::{ fs::File, io::BufWriter, @@ -102,32 +100,29 @@ fn run_math(proof_mode: ProofMode) -> Result<(TimingSummary, usize)> { Duration::ZERO }; - let timing = TimingSummary::new( - ProcessTimings { - commands_other: proof_postprocessing, - ..ProcessTimings::default() - }, - RunTimings { - rulesets: vec![RulesetTiming { - name: "".into(), - role: RulesetTimingRole::Program, - assembly: Duration::ZERO, - pre_merge: PreMergeTiming::Split { - search: Duration::from_nanos(seconds_to_ns(report.search_time)), - apply: Duration::from_nanos(seconds_to_ns(report.apply_time)), - unattributed: Duration::from_nanos(seconds_to_ns( - (report.total_time - - report.search_time - - report.apply_time - - report.rebuild_time) - .max(0.0), - )), - }, - merge: Duration::ZERO, - }], - native_rebuild: Duration::from_nanos(seconds_to_ns(report.rebuild_time)), - }, - )?; + let timing = TimingSummary { + schema_version: TimingSummary::SCHEMA_VERSION, + typecheck_ns: 0, + frontend_parse_ns: 0, + frontend_other_ns: 0, + frontend_install_ns: 0, + commands_actions_ns: 0, + commands_check_ns: 0, + commands_other_ns: proof_postprocessing.as_nanos().min(u64::MAX as u128) as u64, + native_rebuild_ns: seconds_to_ns(report.rebuild_time), + rulesets: vec![RulesetTimingRecord { + name: String::new(), + role: RulesetTimingRole::Program, + assembly_ns: 0, + search_ns: seconds_to_ns(report.search_time), + apply_ns: seconds_to_ns(report.apply_time), + execution_ns: seconds_to_ns( + (report.total_time - report.search_time - report.apply_time - report.rebuild_time) + .max(0.0), + ), + merge_ns: 0, + }], + }; Ok((timing, report.egraph_nodes)) } diff --git a/egglog-experimental/tests/scheduler_reporting.rs b/egglog-experimental/tests/scheduler_reporting.rs index 178d5c50..838c0436 100644 --- a/egglog-experimental/tests/scheduler_reporting.rs +++ b/egglog-experimental/tests/scheduler_reporting.rs @@ -3,6 +3,7 @@ use egglog::CommandOutput; use egglog_reports::RunReport; +use std::collections::BTreeSet; const PROGRAM: &str = r#" (ruleset grow) @@ -17,10 +18,11 @@ const PROGRAM: &str = r#" fn ruleset_names(report: &RunReport) -> Vec { report - .timings() - .rulesets + .iterations .iter() - .map(|timing| timing.name.to_string()) + .map(|iteration| iteration.name.to_string()) + .collect::>() + .into_iter() .collect() } diff --git a/egglog/egglog-reports/src/lib.rs b/egglog/egglog-reports/src/lib.rs index e1135290..1fdfb494 100644 --- a/egglog/egglog-reports/src/lib.rs +++ b/egglog/egglog-reports/src/lib.rs @@ -148,62 +148,8 @@ pub enum RulesetTimingRole { Equality, } -/// Exclusive process work outside ruleset execution. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct ProcessTimings { - pub typecheck: Duration, - pub frontend_parse: Duration, - pub frontend_other: Duration, - pub frontend_install: Duration, - pub commands_actions: Duration, - pub commands_check: Duration, - pub commands_other: Duration, -} - -impl ProcessTimings { - pub fn total(self) -> Duration { - [ - self.typecheck, - self.frontend_parse, - self.frontend_other, - self.frontend_install, - self.commands_actions, - self.commands_check, - self.commands_other, - ] - .into_iter() - .sum() - } -} - -/// Aggregated own-work timing for all iterations of one ruleset. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RulesetTiming { - pub name: Arc, - pub role: RulesetTimingRole, - /// Building the executable ruleset for each invocation, including lazy - /// cached-plan creation on first use. - pub assembly: Duration, - /// Execution before staged updates are merged. - pub pre_merge: PreMergeTiming, - /// Resolving and installing staged updates. - pub merge: Duration, -} - -impl RulesetTiming { - fn add_iteration(&mut self, iteration: &IterationReport) { - self.assembly += iteration.assembly_time; - self.pre_merge.union(iteration.rule_set_report.pre_merge); - self.merge += iteration.rule_set_report.merge_time; - } -} - -/// Derived timing view over a run's annotated iterations. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct RunTimings { - pub rulesets: Vec, - pub native_rebuild: Duration, -} +type AggregatedRulesetTimings = + BTreeMap<(RulesetTimingRole, Arc), (Duration, PreMergeTiming, Duration)>; impl RuleSetReport { pub fn num_matches(&self, rule: &str) -> usize { @@ -266,8 +212,8 @@ pub struct RulesetIteration { /// information together. #[derive(Debug, Serialize, Clone)] pub struct RunReport { - // Since iteration reports are immutable, they are reference counted to - // avoid expensive cloning when e-graphs are cloned. + // Each entry carries ruleset metadata; its immutable report payload remains + // reference-counted so cloning e-graphs stays cheap. pub iterations: Vec, /// If any changes were made to the database. pub updated: bool, @@ -304,11 +250,11 @@ impl Display for RunReport { )?; } - let timings = self.timings(); - for timing in &timings.rulesets { - let assembly_time = timing.assembly.as_secs_f64(); - let merge_time = timing.merge.as_secs_f64(); - match timing.pre_merge { + let (rulesets, native_rebuild) = self.aggregate_timings(); + for ((_, name), (assembly, pre_merge, merge)) in rulesets { + let assembly_time = assembly.as_secs_f64(); + let merge_time = merge.as_secs_f64(); + match pre_merge { PreMergeTiming::Split { search, apply, @@ -317,7 +263,7 @@ impl Display for RunReport { writeln!( f, "Ruleset {}: assembly {assembly_time:.3}s, search {:.3}s, apply {:.3}s, unattributed {:.3}s, merge {merge_time:.3}s", - timing.name, + name, search.as_secs_f64(), apply.as_secs_f64(), unattributed.as_secs_f64(), @@ -327,17 +273,13 @@ impl Display for RunReport { writeln!( f, "Ruleset {}: assembly {assembly_time:.3}s, pre-merge {:.3}s, merge {merge_time:.3}s", - timing.name, + name, elapsed.as_secs_f64(), )?; } } } - writeln!( - f, - "Native rebuild: {:.3}s", - timings.native_rebuild.as_secs_f64() - )?; + writeln!(f, "Native rebuild: {:.3}s", native_rebuild.as_secs_f64())?; Ok(()) } @@ -405,31 +347,29 @@ impl RunReport { } /// Derive the ruleset-own-work and global rebuild partition of this run. - pub fn timings(&self) -> RunTimings { - let mut rulesets = BTreeMap::<(RulesetTimingRole, Arc), RulesetTiming>::new(); + fn aggregate_timings(&self) -> (AggregatedRulesetTimings, Duration) { + let mut rulesets = AggregatedRulesetTimings::new(); let mut native_rebuild = Duration::ZERO; for iteration in &self.iterations { native_rebuild = native_rebuild.saturating_add(iteration.report.rebuild_time); let key = (iteration.role, iteration.name.clone()); match rulesets.entry(key) { std::collections::btree_map::Entry::Occupied(mut entry) => { - entry.get_mut().add_iteration(&iteration.report); + let (assembly, pre_merge, merge) = entry.get_mut(); + *assembly += iteration.report.assembly_time; + pre_merge.union(iteration.report.rule_set_report.pre_merge); + *merge += iteration.report.rule_set_report.merge_time; } std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(RulesetTiming { - name: iteration.name.clone(), - role: iteration.role, - assembly: iteration.report.assembly_time, - pre_merge: iteration.report.rule_set_report.pre_merge, - merge: iteration.report.rule_set_report.merge_time, - }); + entry.insert(( + iteration.report.assembly_time, + iteration.report.rule_set_report.pre_merge, + iteration.report.rule_set_report.merge_time, + )); } } } - RunTimings { - rulesets: rulesets.into_values().collect(), - native_rebuild, - } + (rulesets, native_rebuild) } /// Merge two reports. @@ -441,9 +381,42 @@ impl RunReport { } } +/// All cumulative reporting state owned by an e-graph. +/// +/// Local [`RunReport`] values remain scoped to schedules. Process timing lives +/// here because parsing, typechecking, and command work surrounds those runs +/// and must be counted exactly once. +#[derive(Debug, Clone, Default)] +pub struct OverallReport { + pub run: RunReport, + pub typecheck: Duration, + pub frontend_parse: Duration, + pub frontend_other: Duration, + pub frontend_install: Duration, + pub commands_actions: Duration, + pub commands_check: Duration, + pub commands_other: Duration, +} + +impl OverallReport { + pub fn process_time(&self) -> Duration { + [ + self.typecheck, + self.frontend_parse, + self.frontend_other, + self.frontend_install, + self.commands_actions, + self.commands_check, + self.commands_other, + ] + .into_iter() + .sum() + } +} + /// Compact timing for one ruleset in the benchmark transport. #[derive(Debug, Serialize, Clone, PartialEq, Eq)] -pub struct RulesetTimingSummary { +pub struct RulesetTimingRecord { pub name: String, pub role: RulesetTimingRole, pub assembly_ns: u64, @@ -469,7 +442,7 @@ pub struct TimingSummary { pub commands_check_ns: u64, pub commands_other_ns: u64, pub native_rebuild_ns: u64, - pub rulesets: Vec, + pub rulesets: Vec, } /// A requested timing summary cannot satisfy the serial, single-role contract. @@ -496,52 +469,52 @@ impl Display for TimingSummaryError { impl std::error::Error for TimingSummaryError {} impl TimingSummary { - pub fn new(process: ProcessTimings, mut run: RunTimings) -> Result { - run.rulesets.sort_unstable_by(|left, right| { - (left.role, &left.name).cmp(&(right.role, &right.name)) - }); + pub const SCHEMA_VERSION: u32 = 4; + + pub fn from_report(report: &OverallReport) -> Result { + let (timings, native_rebuild) = report.run.aggregate_timings(); let mut roles = BTreeMap::new(); - let mut rulesets = Vec::with_capacity(run.rulesets.len()); - for timing in run.rulesets { + let mut rulesets = Vec::with_capacity(timings.len()); + for ((role, name), (assembly, pre_merge, merge)) in timings { if roles - .insert(timing.name.clone(), timing.role) - .is_some_and(|role| role != timing.role) + .insert(name.clone(), role) + .is_some_and(|previous| previous != role) { return Err(TimingSummaryError::InconsistentRulesetRole { - ruleset: timing.name.to_string(), + ruleset: name.to_string(), }); } let PreMergeTiming::Split { search, apply, unattributed, - } = timing.pre_merge + } = pre_merge else { return Err(TimingSummaryError::PhaseTimingUnavailable { - ruleset: timing.name.to_string(), + ruleset: name.to_string(), }); }; - rulesets.push(RulesetTimingSummary { - name: timing.name.to_string(), - role: timing.role, - assembly_ns: duration_ns(timing.assembly), + rulesets.push(RulesetTimingRecord { + name: name.to_string(), + role, + assembly_ns: duration_ns(assembly), search_ns: duration_ns(search), apply_ns: duration_ns(apply), execution_ns: duration_ns(unattributed), - merge_ns: duration_ns(timing.merge), + merge_ns: duration_ns(merge), }); } Ok(Self { - schema_version: 4, - typecheck_ns: duration_ns(process.typecheck), - frontend_parse_ns: duration_ns(process.frontend_parse), - frontend_other_ns: duration_ns(process.frontend_other), - frontend_install_ns: duration_ns(process.frontend_install), - commands_actions_ns: duration_ns(process.commands_actions), - commands_check_ns: duration_ns(process.commands_check), - commands_other_ns: duration_ns(process.commands_other), - native_rebuild_ns: duration_ns(run.native_rebuild), + schema_version: Self::SCHEMA_VERSION, + typecheck_ns: duration_ns(report.typecheck), + frontend_parse_ns: duration_ns(report.frontend_parse), + frontend_other_ns: duration_ns(report.frontend_other), + frontend_install_ns: duration_ns(report.frontend_install), + commands_actions_ns: duration_ns(report.commands_actions), + commands_check_ns: duration_ns(report.commands_check), + commands_other_ns: duration_ns(report.commands_other), + native_rebuild_ns: duration_ns(native_rebuild), rulesets, }) } @@ -563,76 +536,65 @@ mod tests { } } + fn iteration( + assembly: u64, + pre_merge: PreMergeTiming, + merge: u64, + rebuild: Duration, + ) -> IterationReport { + IterationReport { + assembly_time: Duration::from_nanos(assembly), + rule_set_report: RuleSetReport { + pre_merge, + merge_time: Duration::from_nanos(merge), + ..RuleSetReport::default() + }, + rebuild_time: rebuild, + } + } + #[test] fn run_report_aggregates_every_iteration_of_a_ruleset() { - let mut report = RunReport::default(); - report.add_iteration( + let mut report = OverallReport::default(); + report.run.add_iteration( "timed", RulesetTimingRole::Program, - IterationReport { - rule_set_report: RuleSetReport { - pre_merge: split(11, 7, 3), - merge_time: Duration::from_nanos(13), - ..RuleSetReport::default() - }, - rebuild_time: Duration::from_nanos(17), - assembly_time: Duration::from_nanos(2), - }, + iteration(2, split(11, 7, 3), 13, Duration::from_nanos(17)), ); - report.add_iteration( + report.run.add_iteration( "timed", RulesetTimingRole::Program, - IterationReport { - rule_set_report: RuleSetReport { - pre_merge: split(19, 5, 4), - merge_time: Duration::from_nanos(23), - ..RuleSetReport::default() - }, - rebuild_time: Duration::from_nanos(29), - assembly_time: Duration::from_nanos(3), - }, + iteration(3, split(19, 5, 4), 23, Duration::from_nanos(29)), ); - let timings = report.timings(); - assert_eq!( - timings.rulesets[0].pre_merge.total(), - Duration::from_nanos(49) - ); - assert_eq!(timings.rulesets[0].assembly, Duration::from_nanos(5)); - assert_eq!(timings.rulesets[0].merge, Duration::from_nanos(36)); - assert_eq!(timings.native_rebuild, Duration::from_nanos(46)); + let summary = TimingSummary::from_report(&report).unwrap(); + assert_eq!(summary.rulesets[0].assembly_ns, 5); + assert_eq!(summary.rulesets[0].search_ns, 30); + assert_eq!(summary.rulesets[0].apply_ns, 12); + assert_eq!(summary.rulesets[0].execution_ns, 7); + assert_eq!(summary.rulesets[0].merge_ns, 36); + assert_eq!(summary.native_rebuild_ns, 46); } #[test] fn timing_summary_exact_json_is_dense_and_sorted() { - let summary = TimingSummary::new( - ProcessTimings { - typecheck: Duration::from_nanos(2), - frontend_parse: Duration::from_nanos(1), - commands_check: Duration::from_nanos(6), - ..ProcessTimings::default() - }, - RunTimings { - rulesets: vec![ - RulesetTiming { - name: "@parent".into(), - role: RulesetTimingRole::Equality, - assembly: Duration::from_nanos(8), - pre_merge: split(9, 10, 11), - merge: Duration::from_nanos(12), - }, - RulesetTiming { - name: "rules/λ".into(), - role: RulesetTimingRole::Program, - assembly: Duration::ZERO, - pre_merge: split(1_000_000_234, 3, 4), - merge: Duration::from_nanos(5), - }, - ], - native_rebuild: Duration::from_nanos(13), - }, - ) - .unwrap(); + let mut report = OverallReport { + typecheck: Duration::from_nanos(2), + frontend_parse: Duration::from_nanos(1), + commands_check: Duration::from_nanos(6), + ..OverallReport::default() + }; + report.run.add_iteration( + "@parent", + RulesetTimingRole::Equality, + iteration(8, split(9, 10, 11), 12, Duration::from_nanos(13)), + ); + report.run.add_iteration( + "rules/λ", + RulesetTimingRole::Program, + iteration(0, split(1_000_000_234, 3, 4), 5, Duration::ZERO), + ); + let summary = TimingSummary::from_report(&report).unwrap(); assert_eq!( serde_json::to_string(&summary).unwrap(), @@ -642,7 +604,7 @@ mod tests { #[test] fn timing_summary_empty_report_golden() { - let summary = TimingSummary::new(ProcessTimings::default(), RunTimings::default()).unwrap(); + let summary = TimingSummary::from_report(&OverallReport::default()).unwrap(); assert_eq!( serde_json::to_string(&summary).unwrap(), r#"{"schema_version":4,"typecheck_ns":0,"frontend_parse_ns":0,"frontend_other_ns":0,"frontend_install_ns":0,"commands_actions_ns":0,"commands_check_ns":0,"commands_other_ns":0,"native_rebuild_ns":0,"rulesets":[]}"# @@ -651,23 +613,24 @@ mod tests { #[test] fn timing_summary_does_not_truncate_rulesets_and_saturates_nanoseconds() { - let summary = TimingSummary::new( - ProcessTimings::default(), - RunTimings { - rulesets: (0..40) - .rev() - .map(|index| RulesetTiming { - name: format!("ruleset-{index:02}").into(), - role: RulesetTimingRole::Program, - assembly: Duration::ZERO, - pre_merge: split(index + 1, 0, 0), - merge: Duration::ZERO, - }) - .collect(), - native_rebuild: Duration::from_secs(u64::MAX), - }, - ) - .unwrap(); + let mut report = OverallReport::default(); + for index in (0_u64..40).rev() { + report.run.add_iteration( + &format!("ruleset-{index:02}"), + RulesetTimingRole::Program, + iteration( + 0, + split(index + 1, 0, 0), + 0, + if index == 0 { + Duration::from_secs(u64::MAX) + } else { + Duration::ZERO + }, + ), + ); + } + let summary = TimingSummary::from_report(&report).unwrap(); assert_eq!(summary.rulesets.len(), 40); assert_eq!(summary.rulesets.first().unwrap().name, "ruleset-00"); @@ -677,21 +640,20 @@ mod tests { #[test] fn timing_summary_rejects_combined_timing_and_inconsistent_roles() { - let combined = TimingSummary::new( - ProcessTimings::default(), - RunTimings { - rulesets: vec![RulesetTiming { - name: "mixed".into(), - role: RulesetTimingRole::Program, - assembly: Duration::ZERO, - pre_merge: PreMergeTiming::Combined { - elapsed: Duration::from_nanos(5), - }, - merge: Duration::ZERO, - }], - native_rebuild: Duration::ZERO, - }, + let mut combined_report = OverallReport::default(); + combined_report.run.add_iteration( + "mixed", + RulesetTimingRole::Program, + iteration( + 0, + PreMergeTiming::Combined { + elapsed: Duration::from_nanos(5), + }, + 0, + Duration::ZERO, + ), ); + let combined = TimingSummary::from_report(&combined_report); assert_eq!( combined, Err(TimingSummaryError::PhaseTimingUnavailable { @@ -699,23 +661,18 @@ mod tests { }) ); - let duplicate = |role| RulesetTiming { - name: "mixed".into(), - role, - assembly: Duration::ZERO, - pre_merge: split(0, 0, 0), - merge: Duration::ZERO, - }; - let inconsistent = TimingSummary::new( - ProcessTimings::default(), - RunTimings { - rulesets: vec![ - duplicate(RulesetTimingRole::Program), - duplicate(RulesetTimingRole::Equality), - ], - native_rebuild: Duration::ZERO, - }, + let mut inconsistent_report = OverallReport::default(); + inconsistent_report.run.add_iteration( + "mixed", + RulesetTimingRole::Program, + iteration(0, split(0, 0, 0), 0, Duration::ZERO), + ); + inconsistent_report.run.add_iteration( + "mixed", + RulesetTimingRole::Equality, + iteration(0, split(0, 0, 0), 0, Duration::ZERO), ); + let inconsistent = TimingSummary::from_report(&inconsistent_report); assert_eq!( inconsistent, Err(TimingSummaryError::InconsistentRulesetRole { diff --git a/egglog/src/lib.rs b/egglog/src/lib.rs index c4b47826..b3e8115e 100644 --- a/egglog/src/lib.rs +++ b/egglog/src/lib.rs @@ -43,7 +43,7 @@ use egglog_bridge::{ColumnTy, QueryEntry}; use egglog_core_relations as core_relations; use egglog_numeric_id as numeric_id; use egglog_reports::{ - ProcessTimings, ReportLevel, RulesetTimingRole, RunReport, TimingSummary, TimingSummaryError, + OverallReport, ReportLevel, RulesetTimingRole, RunReport, TimingSummary, TimingSummaryError, }; pub use exec_state::{ Context, Core, Enode, FullState, FunctionEntry, PureState, Read, ReadState, Write, WriteState, @@ -325,10 +325,8 @@ pub struct EGraph { pub seminaive: bool, pub no_decomp: bool, type_info: TypeInfo, - /// The run report unioned over all runs so far. - overall_run_report: RunReport, - /// Exclusive process work outside ruleset execution. - process_timings: ProcessTimings, + /// Cumulative run and process reporting state. + overall_report: OverallReport, schedulers: DenseIdMap, commands: IndexMap>, extension_state: HashMap>, @@ -447,8 +445,7 @@ impl EGraph { fact_directory: None, seminaive: true, no_decomp: false, - overall_run_report: Default::default(), - process_timings: Default::default(), + overall_report: Default::default(), type_info: Default::default(), schedulers: Default::default(), commands: Default::default(), @@ -850,10 +847,8 @@ impl EGraph { pub fn pop(&mut self) -> Result<(), Error> { match self.pushed_egraph.take() { Some(mut e) => { - // Preserve the overall report from the popped egraph - std::mem::swap(&mut self.overall_run_report, &mut e.overall_run_report); // Work performed in the popped scope still belongs to this run. - std::mem::swap(&mut self.process_timings, &mut e.process_timings); + std::mem::swap(&mut self.overall_report, &mut e.overall_report); // Preserve the symbol generator so that fresh symbols // generated after pop don't collide with ones generated before pop. std::mem::swap(&mut self.parser.symbol_gen, &mut e.parser.symbol_gen); @@ -1407,7 +1402,7 @@ impl EGraph { self.rulesets[ruleset].timing_role, iteration_report, ); - self.overall_run_report.union(report.clone()); + self.overall_report.run.union(report.clone()); Ok(report) } @@ -1997,7 +1992,7 @@ impl EGraph { self.backend.free_rule(id); self.backend.free_external_func(ext_id); let iteration_report = run_result.map_err(|e| Error::BackendError(e.to_string()))?; - self.process_timings.commands_check += iteration_report.total_time(); + self.overall_report.commands_check += iteration_report.total_time(); let ext_sc_val = ext_sc.lock().unwrap().take(); let matched = matches!(ext_sc_val, Some(())); @@ -2034,11 +2029,14 @@ impl EGraph { _ => CommandPhase::Other, }; let command_timer = Instant::now(); - let process_before = self.process_timings.total(); - let iteration_before = self.overall_run_report.iterations.len(); + let process_before = self.overall_report.process_time(); + let iteration_before = self.overall_report.run.iterations.len(); let result = self.run_command_inner(command); - let nested_process = self.process_timings.total().saturating_sub(process_before); - let nested_rulesets = self.overall_run_report.iterations[iteration_before..] + let nested_process = self + .overall_report + .process_time() + .saturating_sub(process_before); + let nested_rulesets = self.overall_report.run.iterations[iteration_before..] .iter() .map(|iteration| iteration.report.total_time()) .sum(); @@ -2046,10 +2044,10 @@ impl EGraph { .elapsed() .saturating_sub(nested_process + nested_rulesets); match phase { - CommandPhase::Install => self.process_timings.frontend_install += own_time, - CommandPhase::Actions => self.process_timings.commands_actions += own_time, - CommandPhase::Check => self.process_timings.commands_check += own_time, - CommandPhase::Other => self.process_timings.commands_other += own_time, + CommandPhase::Install => self.overall_report.frontend_install += own_time, + CommandPhase::Actions => self.overall_report.commands_actions += own_time, + CommandPhase::Check => self.overall_report.commands_check += own_time, + CommandPhase::Other => self.overall_report.commands_other += own_time, } result } @@ -2124,7 +2122,7 @@ impl EGraph { None => { log::info!("Printed overall statistics"); return Ok(vec![CommandOutput::OverallStatistics( - self.overall_run_report.clone(), + self.overall_report.run.clone(), )]); } Some(path) => { @@ -2132,7 +2130,7 @@ impl EGraph { .map_err(|e| Error::IoError(path.clone().into(), e, span.clone()))?; log::info!("Printed overall statistics to json file {path}"); - serde_json::to_writer(&mut file, &self.overall_run_report).map_err(|e| { + serde_json::to_writer(&mut file, &self.overall_report.run).map_err(|e| { Error::BackendError(format!("failed writing statistics: {e}")) })?; } @@ -2629,7 +2627,7 @@ impl EGraph { // TODO this is ugly- we don't need an entire e-graph just for type information. let typecheck_timer = Instant::now(); let typechecked = original_typechecking.typecheck_program(&desugared)?; - self.process_timings.typecheck += typecheck_timer.elapsed(); + self.overall_report.typecheck += typecheck_timer.elapsed(); for command in &typechecked { if let Err(reason) = command_supports_proof_encoding( @@ -2648,7 +2646,7 @@ impl EGraph { } else { let typecheck_timer = Instant::now(); let mut typechecked = self.typecheck_program(&desugared)?; - self.process_timings.typecheck += typecheck_timer.elapsed(); + self.overall_report.typecheck += typecheck_timer.elapsed(); typechecked = remove_globals::remove_globals(typechecked, &mut self.parser.symbol_gen); for command in &typechecked { @@ -2663,10 +2661,13 @@ impl EGraph { /// When will_run is true, adds to `desugared_commands_run_so_far`, which is used for proof checking. fn resolve_command(&mut self, command: Command) -> Result { let lowering_timer = Instant::now(); - let nested_before = self.process_timings.total(); + let nested_before = self.overall_report.process_time(); let resolved = self.resolve_command_inner(command); - let nested = self.process_timings.total().saturating_sub(nested_before); - self.process_timings.frontend_other += lowering_timer.elapsed().saturating_sub(nested); + let nested = self + .overall_report + .process_time() + .saturating_sub(nested_before); + self.overall_report.frontend_other += lowering_timer.elapsed().saturating_sub(nested); resolved } @@ -2720,7 +2721,7 @@ impl EGraph { // Now typecheck using self, adding term type information. let typecheck_timer = Instant::now(); let desugared_typechecked = self.typecheck_program(&desugared)?; - self.process_timings.typecheck += typecheck_timer.elapsed(); + self.overall_report.typecheck += typecheck_timer.elapsed(); // Remove the globals the term encoding itself introduced (its minted // `let`s), the same way source-level globals were removed above. let desugared_typechecked = remove_globals::remove_globals( @@ -2763,7 +2764,7 @@ impl EGraph { &mut self.parser.symbol_gen, macro_type_info, ); - self.process_timings.frontend_other += macro_timer.elapsed(); + self.overall_report.frontend_other += macro_timer.elapsed(); let macro_expanded = macro_expanded?; for command in macro_expanded { @@ -2772,7 +2773,7 @@ impl EGraph { let include_timer = Instant::now(); let s = std::fs::read_to_string(file) .map_err(|e| Error::IoError(file.clone().into(), e, span.clone())); - self.process_timings.frontend_other += include_timer.elapsed(); + self.overall_report.frontend_other += include_timer.elapsed(); let s = s?; let included_program = self.parse_program_timed(Some(file.clone()), &s)?; // run program internal on these include commands @@ -2865,7 +2866,7 @@ impl EGraph { ) -> Result, Error> { let parse_timer = Instant::now(); let parsed = self.parser.get_program_from_string(filename, input); - self.process_timings.frontend_parse += parse_timer.elapsed(); + self.overall_report.frontend_parse += parse_timer.elapsed(); Ok(parsed?) } @@ -2920,11 +2921,11 @@ impl EGraph { /// Gets the overall run report and returns it. pub fn get_overall_run_report(&self) -> &RunReport { - &self.overall_run_report + &self.overall_report.run } pub(crate) fn timing_summary(&self) -> Result { - TimingSummary::new(self.process_timings, self.overall_run_report.timings()) + TimingSummary::from_report(&self.overall_report) } /// Convert from an egglog value to a Rust type. @@ -3782,12 +3783,12 @@ mod tests { .parse_and_run_program(None, "(datatype Math (Num i64)) (let value (Num 1))") .unwrap(); - assert!(egraph.process_timings.frontend_parse > std::time::Duration::ZERO); - assert!(egraph.process_timings.typecheck > std::time::Duration::ZERO); - assert!(egraph.process_timings.frontend_other > std::time::Duration::ZERO); + assert!(egraph.overall_report.frontend_parse > std::time::Duration::ZERO); + assert!(egraph.overall_report.typecheck > std::time::Duration::ZERO); + assert!(egraph.overall_report.frontend_other > std::time::Duration::ZERO); let source_checker = egraph.proof_state.original_typechecking.as_ref().unwrap(); assert_eq!( - source_checker.process_timings.typecheck, + source_checker.overall_report.typecheck, std::time::Duration::ZERO, "the child checker must not retain time omitted from the outer summary" ); diff --git a/egglog/src/scheduler.rs b/egglog/src/scheduler.rs index f3a6e55c..bd39bb69 100644 --- a/egglog/src/scheduler.rs +++ b/egglog/src/scheduler.rs @@ -310,7 +310,7 @@ impl EGraph { self.schedulers = schedulers; if let Ok(report) = &result { - self.overall_run_report.union(report.clone()); + self.overall_report.run.union(report.clone()); } result } @@ -514,14 +514,12 @@ mod test { .keys() .all(|k| k.starts_with("test-rule")) ); - assert_eq!( + assert!(!report.iterations.is_empty()); + assert!( report - .timings() - .rulesets + .iterations .iter() - .map(|timing| timing.name.as_ref()) - .collect::>(), - ["test"] + .all(|iteration| iteration.name.as_ref() == "test") ); if report.can_stop { From 8d9ea709460b75c271201f4c07573e1972de80f5 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Thu, 13 Aug 2026 13:24:27 -0400 Subject: [PATCH 9/9] Finish timing report integration --- README.md | 4 +++ egglog/CHANGELOG.md | 1 + egglog/egglog-reports/src/lib.rs | 46 ++++++++++++++++++++++++++++- egglog/src/lib.rs | 41 +++++++++++++++++++++++++- egglog/src/scheduler.rs | 3 ++ tests/test_report_rendering.py | 50 ++++++++++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b229d9b8..6121d5b0 100644 --- a/README.md +++ b/README.md @@ -475,6 +475,10 @@ measurements. Successful observations also contain the version-4 timing summary: fixed process counters, a typed list of named ruleset timings, and one global native-Rebuild counter. Changes to timing coverage or meaning require a schema-version change so stale measurements cannot be reused silently. +The experimental custom-scheduler API times its backend query and action +invocations as ruleset work; lazy rule compilation and its intermediate update +flush remain surrounding work and are charged to an enclosing command when one +exists. Timed-out rows have null wall time, peak RSS, and timing summary. Failed rows have no timing summary and retain whatever process measurements the operating diff --git a/egglog/CHANGELOG.md b/egglog/CHANGELOG.md index 0a1f9728..58ae002a 100644 --- a/egglog/CHANGELOG.md +++ b/egglog/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] - ReleaseDate +- **Breaking reporting formats.** `--save-report` now stores each iteration with its ruleset name and timing responsibility, and no longer serializes the redundant `ruleset_timings` or `search_and_apply_time_per_rule` aggregates. `--timing-summary` now emits the typed version-4 timing partition used by the benchmark runner; the runner's JSONL schema is also version 4, so older disposable benchmark caches must be recomputed. - **Proof mode is substantially faster and uses less memory.** The term/proof encoding no longer writes each proof's `Congr`/`Trans`/`Sym` steps as rows while rules run; it records what justified a fact and rebuilds the steps when a proof is asked for. A flat `(rewrite (Add a b) (Add b a))` firing writes 2 proof rows where it wrote 13. Across the benchmark suite that is 0.73–0.75x wall time, and peak memory on `math-microbenchmark` goes from 2.3 GiB to 1.1 GiB. Proof semantics are unchanged by this work; the proof snapshots that do move in this release move for the separate fixes and the now-deterministic extraction order below. - Fix `set-if-empty` in the term/proof encoding looking its key up in the committed table while staging its insert, so two calls with the same key in one action batch both missed and both inserted, minting two e-classes for one term — leaving the encoding one iteration behind ordinary execution on programs that do not saturate. It now reads through the batch's predicted rows, as `lookup_or_insert` already did. - **Declared indexes.** `(index (any *))` declares a read-only relation over the rows of ``: each value appearing in *any* of the listed columns, followed by the whole row. It answers "which rows mention this value", which no ordinary atom expresses — repeating a variable across those columns instead constrains them to be equal. Purely physical: it changes no results, only the cost of finding those rows. The database maintains it and creates it on demand, so the declaration binds a name and nothing more. An atom is probed rather than scanned, so a *variable* indexed value must be bound elsewhere in the query by a function's rows — a body primitive runs after the join, so it cannot bind one — while a literal is already known and needs no binder. Over a single indexed column the occurrence is an ordinary equality and the atom is lowered to a plain one, which is also what lets the indexed value sit at another of the row's columns; over several columns that combination is a per-row disjunction and is rejected. A *user-written* declaration is not yet supported under the term/proof encoding, which rewrites a function into a view whose columns differ, so the declaration would name the wrong ones; the encoder's own declarations, written against its views, are unaffected. `(any …)` is an extractor over the row, leaving room for others (e.g. the elements of a container column). diff --git a/egglog/egglog-reports/src/lib.rs b/egglog/egglog-reports/src/lib.rs index 1fdfb494..67aa6e35 100644 --- a/egglog/egglog-reports/src/lib.rs +++ b/egglog/egglog-reports/src/lib.rs @@ -576,6 +576,45 @@ mod tests { assert_eq!(summary.native_rebuild_ns, 46); } + #[test] + fn run_report_preserves_mixed_pre_merge_totals() { + let mut report = RunReport::default(); + report.add_iteration( + "mixed", + RulesetTimingRole::Program, + iteration(2, split(3, 5, 7), 11, Duration::from_nanos(13)), + ); + report.add_iteration( + "mixed", + RulesetTimingRole::Program, + iteration( + 17, + PreMergeTiming::Combined { + elapsed: Duration::from_nanos(19), + }, + 23, + Duration::from_nanos(29), + ), + ); + + let (rulesets, native_rebuild) = report.aggregate_timings(); + let ((role, name), (assembly, pre_merge, merge)) = rulesets.iter().next().unwrap(); + assert_eq!(rulesets.len(), 1); + assert_eq!( + (*role, name.as_ref()), + (RulesetTimingRole::Program, "mixed") + ); + assert_eq!(*assembly, Duration::from_nanos(19)); + assert_eq!( + *pre_merge, + PreMergeTiming::Combined { + elapsed: Duration::from_nanos(34) + } + ); + assert_eq!(*merge, Duration::from_nanos(34)); + assert_eq!(native_rebuild, Duration::from_nanos(42)); + } + #[test] fn timing_summary_exact_json_is_dense_and_sorted() { let mut report = OverallReport { @@ -594,11 +633,16 @@ mod tests { RulesetTimingRole::Program, iteration(0, split(1_000_000_234, 3, 4), 5, Duration::ZERO), ); + report.run.add_iteration( + "", + RulesetTimingRole::Program, + iteration(0, split(0, 0, 0), 0, Duration::ZERO), + ); let summary = TimingSummary::from_report(&report).unwrap(); assert_eq!( serde_json::to_string(&summary).unwrap(), - r#"{"schema_version":4,"typecheck_ns":2,"frontend_parse_ns":1,"frontend_other_ns":0,"frontend_install_ns":0,"commands_actions_ns":0,"commands_check_ns":6,"commands_other_ns":0,"native_rebuild_ns":13,"rulesets":[{"name":"rules/λ","role":"program","assembly_ns":0,"search_ns":1000000234,"apply_ns":3,"execution_ns":4,"merge_ns":5},{"name":"@parent","role":"equality","assembly_ns":8,"search_ns":9,"apply_ns":10,"execution_ns":11,"merge_ns":12}]}"# + r#"{"schema_version":4,"typecheck_ns":2,"frontend_parse_ns":1,"frontend_other_ns":0,"frontend_install_ns":0,"commands_actions_ns":0,"commands_check_ns":6,"commands_other_ns":0,"native_rebuild_ns":13,"rulesets":[{"name":"","role":"program","assembly_ns":0,"search_ns":0,"apply_ns":0,"execution_ns":0,"merge_ns":0},{"name":"rules/λ","role":"program","assembly_ns":0,"search_ns":1000000234,"apply_ns":3,"execution_ns":4,"merge_ns":5},{"name":"@parent","role":"equality","assembly_ns":8,"search_ns":9,"apply_ns":10,"execution_ns":11,"merge_ns":12}]}"# ); } diff --git a/egglog/src/lib.rs b/egglog/src/lib.rs index b3e8115e..834bef0f 100644 --- a/egglog/src/lib.rs +++ b/egglog/src/lib.rs @@ -3124,7 +3124,15 @@ impl EGraph { results.push(map); Some(()) })?; - prelude::run_ruleset(self, &ruleset)?; + let rule_ids = match &self.rulesets[&ruleset].kind { + RulesetKind::Rules(rules) => rules.values().map(|(_, id)| *id).collect::>(), + RulesetKind::Combined(_) => unreachable!("the query ruleset was created directly"), + }; + let iteration_report = self + .backend + .run_rules(&rule_ids) + .map_err(|e| Error::BackendError(e.to_string()))?; + self.overall_report.commands_check += iteration_report.total_time(); Ok(()) })(); @@ -3794,6 +3802,37 @@ mod tests { ); } + #[test] + fn query_is_recorded_as_command_work_without_persistent_ruleset_rows() { + let mut egraph = EGraph::default(); + egraph + .parse_and_run_program(None, "(relation R (i64)) (R 1) (R 2)") + .unwrap(); + let iterations_before = egraph.overall_report.run.iterations.len(); + let check_time_before = egraph.overall_report.commands_check; + + for _ in 0..2 { + let matches = egraph + .query(crate::vars![x: i64], crate::facts![(R x)]) + .unwrap(); + assert_eq!(matches.len(), 2); + } + + assert_eq!( + egraph.overall_report.run.iterations.len(), + iterations_before + ); + assert!(egraph.overall_report.commands_check > check_time_before); + assert!( + egraph + .timing_summary() + .unwrap() + .rulesets + .iter() + .all(|ruleset| !ruleset.name.contains("query_ruleset")) + ); + } + #[derive(Clone)] struct InnerProduct { vec: ArcSort, diff --git a/egglog/src/scheduler.rs b/egglog/src/scheduler.rs index bd39bb69..7ca095c1 100644 --- a/egglog/src/scheduler.rs +++ b/egglog/src/scheduler.rs @@ -188,6 +188,9 @@ impl EGraph { /// /// The iteration is recorded in the overall run report, as in /// [`EGraph::step_rules`]. + /// Lazy rule compilation and the intermediate update flush surround the + /// recorded query/action invocations; an enclosing command therefore + /// attributes that work to command timing rather than ruleset timing. pub fn step_rules_with_scheduler( &mut self, scheduler_id: SchedulerId, diff --git a/tests/test_report_rendering.py b/tests/test_report_rendering.py index 87f347d7..79956ff7 100644 --- a/tests/test_report_rendering.py +++ b/tests/test_report_rendering.py @@ -325,6 +325,56 @@ def test_ruleset_detail_unfolds_program_and_equality_with_explicit_children(tmp_ assert rewrite.rows[0].cells[1].tone == "default" +def test_ruleset_edges_label_empty_names_and_break_equal_deltas_by_name(tmp_path: Path) -> None: + report_path = tmp_path / "ruleset-ties.jsonl" + file = models.FileSpec("file.egg", tmp_path / "file.egg", "sha256:file") + baseline = make_endpoint(binary_sha256="sha256:baseline", treatment="off") + candidate = make_endpoint(binary_sha256="sha256:candidate", treatment="proofs") + unchanged = make_ruleset_timing("unchanged", search_ns=0, apply_ns=0, merge_ns=0) + tied_names = ("zeta", "beta", "eta", "delta", "gamma", "alpha", "epsilon") + write_report( + report_path, + make_record( + 0, + started_at="2026-07-17T12:00:00Z", + binary_sha256=baseline.target.binary_sha256, + timing_summary=make_timing_summary(unchanged, native_rebuild_ns=0), + ), + make_record( + 1, + started_at="2026-07-17T12:00:01Z", + binary_sha256=candidate.target.binary_sha256, + treatment="proofs", + wall_sec=1.2, + timing_summary=make_timing_summary( + *(make_ruleset_timing(name, search_ns=10_000_000, apply_ns=0, merge_ns=0) for name in tied_names), + make_ruleset_timing("", role="equality", search_ns=1_000_000, apply_ns=0, merge_ns=0), + unchanged, + native_rebuild_ns=0, + ), + ), + ) + comparison = models.ComparisonSpec(baseline, candidate, (file,), 1, 120) + + catalog = build_report_catalog(ReportStore(report_path), comparison, "rulesets") + section = next(section for section in catalog.sections if section.id == "rulesets") + table = next(block for block in section.blocks if isinstance(block, ReportTable)) + default_ruleset = next(row for row in table.rows if row.cells[0].display == "↳ ") + + assert default_ruleset.cells[0].raw == "" + assert [row.cells[0].display for row in table.rows[:8]] == [ + "Program rules — own work", + "↳ alpha", + "↳ beta", + "↳ delta", + "↳ epsilon", + "↳ eta", + "↳ Other (2 more source rulesets)", + "Equality/rebuild — net", + ] + assert table.rows[6].cells[1].display == "+20.0 ms" + + def test_ratio_tones_use_green_for_improvements_and_dim_unclear_results(tmp_path: Path) -> None: report_path, comparison = _pair_case(tmp_path) catalog = build_report_catalog(ReportStore(report_path), comparison)