diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca966031..4ea98173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,12 @@ jobs: # feature) in automated runs so it keeps guarding against a pass that # produces an ill-typed tree — it is off by default locally because it is # superlinear on nested comprehensions (see `ci_test` / `debug_typecheck`). - run: DEEP_TYPECHECK=1 ./ci.sh test + # `CAMBRA_LINEAGE_GATE=1` makes every compile fold its pane boundaries and + # gate the leak classes, so the gate's corpus is the whole test suite + # rather than the handful of programs `context.rs`'s `corpus()` lists. Two + # recording gaps lived outside that sample; measured at ~+4% on the test + # step, which is worth paying to keep the next one from doing the same. + run: DEEP_TYPECHECK=1 CAMBRA_LINEAGE_GATE=1 ./ci.sh test # 4. Success Signal for noops - name: No-op for docs diff --git a/NodeId b/NodeId new file mode 100644 index 00000000..e69de29b diff --git a/src/ccl/ccl_utils.rs b/src/ccl/ccl_utils.rs index 914d87b9..546b8f36 100644 --- a/src/ccl/ccl_utils.rs +++ b/src/ccl/ccl_utils.rs @@ -338,7 +338,7 @@ pub fn apply_function(expr: Expr, function: Expr, output_ty: Type) -> Expr { /// where its own guard holds and no earlier arm's did. `prior` holds `g₀ … gᵢ₋₁` /// in order; `guard` is `gᵢ`. Every guard is a `Bool`-typed expression over the /// same element, so the synthesized conjunction is `Bool` too — typed here -/// because the callers (post-inference desugar, lambda elimination, the +/// because the callers (post-inference channelize, lambda elimination, the /// transaction path walk) must be type-preserving. /// /// Shared by [`crate::ccl::channelize`] (feed fan-out), [`crate::ccl::lambda_elim`] @@ -1255,10 +1255,41 @@ impl Default for PredMemo { PredMemo(Rc::new(RefCell::new(MemoStore { entries: HashMap::new(), revision: 0, + replacing: false, }))) } } +impl PredMemo { + /// A memo whose rebuilds **replace** the terms they are handed, so the + /// rebuilt term keeps the original's `NodeId`s. + /// + /// Only sound when the owning walk reaches **every** occurrence of every + /// predicate it rebuilds. A predicate is an `Rc` shared across many type + /// slots and a rebuild cannot mutate through it, so it builds a new `Rc` and + /// repoints the refinement it was given. If some other type still holds the + /// original, the two coexist — and preserving ids then puts one id-set on two + /// live terms, which nothing catches because predicate uniqueness is not + /// asserted. + /// + /// [`uniquify`](crate::ccl::uniquify) is the one caller entitled to this: it + /// walks the whole tree, and asserts the resulting 1:1 correspondence + /// (N distinct terms in, N out, same ids) on every compile. + /// + /// Every other caller rebuilds within a single type while the original may + /// survive elsewhere. Those are *derivations*, and [`Default`] gives them the + /// recording behaviour they need. + pub fn replacing() -> Self { + let m = Self::default(); + m.0.borrow_mut().replacing = true; + m + } + + fn is_replacing(&self) -> bool { + self.0.borrow().replacing + } +} + /// Shares predicate *terms* without sharing rebuild *results*: the transform runs /// at every occurrence, and occurrences that entered sharing one `Rc` leave /// sharing one `Rc`. @@ -1289,6 +1320,9 @@ struct MemoStore { /// own recursion, which the callback cannot report: a nested reuse mutates the /// callback's copy without the callback doing anything. revision: u64, + /// See [`PredMemo::replacing`]: the owning walk reaches every occurrence, so + /// a rebuild is a replacement and keeps the original's ids. + replacing: bool, } struct Entry { @@ -1342,7 +1376,7 @@ impl PredMemo { context: &C, f: impl FnOnce(&mut Expr) -> bool, ) -> bool { - let (mut pred, keepalive, before) = { + let (keepalive, before) = { let mut store = self.0.borrow_mut(); let hit = store .entries @@ -1356,22 +1390,39 @@ impl PredMemo { } None => { let keepalive = Rc::clone(&refinement.predicate); - // Copy-on-write, not duplication: the rebuilt term is - // installed *in place of* the original, so it is the same - // logical node at a new allocation and keeps its ids. - let copy = refinement.predicate.clone_preserving_ids(); let rev = store.revision; - (copy, keepalive, rev) + (keepalive, rev) } } }; - // The rebuild runs id-preserving, covering the rewrite as well as the - // copy-on-write above (a substitution firing inside a predicate - // materializes its template here). Nothing records a predicate rewrite, - // so an id minted here is one no record explains; preserving is honest - // because the rebuilt term *replaces* the original everywhere this walk - // reaches. - let reported = crate::ccl::lineage::preserving_ids(|| f(&mut pred)); + // The copy and the rewrite run together under the memo's declared intent + // — `f` mints and copies *into* the term, so its products belong to + // whichever the rebuild is. + // + // **Replacing** (see [`PredMemo::replacing`]): the rebuilt term stands in + // for the original everywhere, so it is the same logical predicate at a + // new allocation and keeps its ids. + // + // **Deriving** (the default): the original may survive on a type this + // walk never reaches, so the rebuilt term is a genuinely new one. It + // freshens, recorded against the source predicate's own root, which records + // it as derived from the term it was rebuilt from. + let (pred, reported) = if self.is_replacing() { + crate::ccl::lineage::preserving_ids(|| { + let mut pred = (*keepalive).clone(); + let reported = f(&mut pred); + (pred, reported) + }) + } else { + let _g = crate::ccl::lineage::enter( + keepalive.node_id(), + "predicate.rebuild", + crate::ccl::lineage::Nature::Machinery, + ); + let mut pred = (*keepalive).clone(); + let reported = f(&mut pred); + (pred, reported) + }; let mut store = self.0.borrow_mut(); let changed = reported || store.revision != before; let installed = if changed { @@ -1407,10 +1458,23 @@ impl TermMemo { /// caller) still leaves the occurrence rebuilt and recorded. pub fn rebuild_always(&self, refinement: &mut Refinement, f: impl FnOnce(&mut Expr)) { let keepalive = Rc::clone(&refinement.predicate); - // Copy-on-write; see `rebuild`. - let mut pred = refinement.predicate.clone_preserving_ids(); - // Id-preserving; see `rebuild`. - crate::ccl::lineage::preserving_ids(|| f(&mut pred)); + // Copy and rewrite under the memo's declared intent; see `rebuild`. + let pred = if self.0.is_replacing() { + crate::ccl::lineage::preserving_ids(|| { + let mut pred = (*keepalive).clone(); + f(&mut pred); + pred + }) + } else { + let _g = crate::ccl::lineage::enter( + keepalive.node_id(), + "predicate.rebuild", + crate::ccl::lineage::Nature::Machinery, + ); + let mut pred = (*keepalive).clone(); + f(&mut pred); + pred + }; let mut store = self.0.0.borrow_mut(); let shared = store .entries diff --git a/src/ccl/channelize.rs b/src/ccl/channelize.rs index 6a664063..32502604 100644 --- a/src/ccl/channelize.rs +++ b/src/ccl/channelize.rs @@ -64,7 +64,7 @@ //! # Transformation (cluster algorithm) //! //! For a cluster of consecutive `let d_i = Defer in …` bindings, -//! `desugar` performs three steps: +//! `channelize_expr` performs three steps: //! //! 1. **Feed extraction.** Walk the cluster body and collect every //! `Feed(d_i, V)` / `Define(d_i, V)` plus the iteration context @@ -116,6 +116,7 @@ use crate::ccl::{ TypedExpr, TypedExprNode, ccl_utils::{count_free, synthesize_arm_predicate, typed_compose}, letrec::check_letrec_causal, + lineage, }; /// `true` when `ty` carries channelization-erasable residue — a `Hole` stamped @@ -145,7 +146,7 @@ fn has_type_residue(ty: &Type) -> bool { } } -/// Errors that can arise while desugaring `Defer`/`Feed`/`Define` nodes. +/// Errors that can arise while channelizing `Defer`/`Feed`/`Define` nodes. #[derive(Debug, PartialEq)] pub enum DeferError { /// A deferred binding had no corresponding `Feed` or `Define` in its scope. @@ -213,7 +214,7 @@ impl fmt::Display for DeferError { } /// Recognize a guard-only `Case` that feeds `defer_name` in one or more arms — -/// the desugar-stage counterpart of `lambda_elim`'s `is_filter_case_body`. +/// the channelize-stage counterpart of `lambda_elim`'s `is_filter_case_body`. /// /// Shape (as lowered from `if g₀: d << v₀ elif g₁: d << v₁ … [else: d << vₑ]` in /// a for-loop body): `Case { None, [g₀ → body₀; …; true → bodyₜ] }`, where each @@ -306,7 +307,7 @@ fn is_lift_shape(bound_expr: &Expr) -> bool { /// /// Rewrites to: `let y = Defer in body_x[x → y] with Var(y) replaced /// by body_y`. The substitution `x → Var(y)` is done via -/// [`desugar_substitute`], which also renames the *target* name of +/// [`channelize_substitute`], which also renames the *target* name of /// `Feed`/`Define` nodes when the replacement is a `Var` — so /// `Feed("x", …)` becomes `Feed("y", …)` automatically. /// @@ -360,7 +361,7 @@ fn lift_defer(binding_name: &Name, bound_expr: Expr, body: &Expr) -> DeferLift { (inner_binding.name, inner_be.ty, *inner_body); // `body_x[x → y]` — also renames Feed/Define targets named `x` to `y`. - let inner_subst = desugar_rename(inner_body_x, &inner_name, binding_name); + let inner_subst = channelize_rename(inner_body_x, &inner_name, binding_name); // Wrap `body_y` with the prefix (renaming stale feed targets to `y`). let mut new_outer_body = body.clone(); @@ -472,7 +473,7 @@ fn contains_defer(expr: &Expr) -> bool { /// `Define(target, …)` node where `target == name`, respecting shadowing /// by `Let`/`Lambda` bindings that rebind `name`. /// -/// Debug-only: the sole caller is the `desugar` invariant assert that a +/// Debug-only: the sole caller is the `channelize_expr` invariant assert that a /// bare-`Var` defer alias never survives `inline` into channelize. #[cfg(debug_assertions)] fn contains_feed_or_define_for(expr: &Expr, name: &Name) -> bool { @@ -505,7 +506,7 @@ fn contains_feed_or_define_for(expr: &Expr, name: &Name) -> bool { /// Substitute every free `Var(name)` in `expr` with `replacement`. /// /// Replace every free occurrence of `Var(name)` with `replacement` during -/// desugaring, renaming `Feed`/`Define` targets along the way: when +/// channelization, renaming `Feed`/`Define` targets along the way: when /// `replacement` is a `Var(new_name)`, handle uses of `name` become /// `new_name` — the α-renaming that makes alias-inlining for defer handles /// correct. @@ -513,11 +514,11 @@ fn contains_feed_or_define_for(expr: &Expr, name: &Name) -> bool { /// A thin wrapper over the uniform engine's in-place mode /// ([`crate::ccl::subst::Subst::rewrite_expr`]). Unlike the pre-port /// version, the engine also rewrites type-carried refinement predicates -/// (`Cast` targets, annotations), so a desugar rename now reaches a +/// (`Cast` targets, annotations), so a channelize rename now reaches a /// predicate that closes over the renamed binder instead of leaving a stale /// reference; and a `Case` pattern binding correctly shadows `name` in its /// branch. -fn desugar_rename(expr: Expr, from: &Name, to: &Name) -> Expr { +fn channelize_rename(expr: Expr, from: &Name, to: &Name) -> Expr { let mut expr = expr; // A **rename**, not a discharge of a bare variable. Both rewrite the same // occurrences, but the species is what carries the occurrence's type onto the @@ -529,8 +530,8 @@ fn desugar_rename(expr: Expr, from: &Name, to: &Name) -> Expr { expr } -/// State threaded through the desugar walk: the channel domains it resolves. -struct DesugarCtx { +/// State threaded through the channelize walk: the channel domains it resolves. +struct ChannelizeCtx { /// each channelized defer's concrete channel domain, /// keyed by its (nominal) `ChanDom` name — recorded as clusters are /// assembled, then closed and substituted over the whole tree by [`run`]. @@ -540,7 +541,7 @@ struct DesugarCtx { resolved_domains: Vec<(Name, Type)>, } -impl DesugarCtx { +impl ChannelizeCtx { fn new() -> Self { Self { resolved_domains: Vec::new(), @@ -568,17 +569,17 @@ impl DesugarCtx { /// (rather than leaving it for `simplify`) means no later pass needs to /// pattern-match `ExprStmt`. pub fn run(expr: Expr) -> Result { - let mut ctx = DesugarCtx::new(); + let mut ctx = ChannelizeCtx::new(); // Cluster channelization. Walks the tree, processes `let d = Defer in …` // clusters, extracting feeds and building each defer's channel. // // Defer-mediating UDFs (`def g(out): out << e`, `def f(n): x = defer(); // …; x`) never reach here: `inline` beta-reduces every such function at its - // call site *before* this pass (it runs pre-desugar — see the `inline` + // call site *before* this pass (it runs pre-channelize — see the `inline` // module docs), leaving only the flattened `let d = Defer in …` chains this // walk handles. The former Phase-1 chain rewriter and the call-site smart // walker that existed for the un-inlined higher-order case are retired. - let rewritten = desugar(expr, &mut ctx)?; + let rewritten = channelize_expr(expr, &mut ctx)?; let mut rewritten = drop_expr_stmts(rewritten); assert_no_defer_residue(&rewritten)?; // With nominal channel domains, every consumer of a defer read typed @@ -587,7 +588,7 @@ pub fn run(expr: Expr) -> Result { // whole-tree type substitution: map each `ChanDom(d)` to its assembled // channel's concrete domain, and erase each `Feed`-kind history to its bare // stream `Fun` — the exact feed-side analog of `mut_elim::erase_mut`. The - // strict post-desugar `typecheck` in `compile_program` backstops the + // strict post-channelize `typecheck` in `compile_program` backstops the // invariant. let mut map = close_chan_domains(std::mem::take(&mut ctx.resolved_domains)); erase_chan_domains(&mut rewritten, &mut map); @@ -809,7 +810,7 @@ fn fun_domain(ty: &Type) -> Option { } /// Debug-only invariant: after [`run`] on a typed input, no expression or -/// binder slot may still carry a `Hole`, `Infer`, or `Feed` type — desugar +/// binder slot may still carry a `Hole`, `Infer`, or `Feed` type — channelize /// erased the defer constructs, so their transient types must be gone too. /// (Refinement predicates are checked by the strict `typecheck` instead; /// walking them here would need the cycle guards it already has.) @@ -882,7 +883,7 @@ fn refine_source_domain(source: &mut Expr, refinement: Refinement) { /// Collapse every `ExprStmt(e, b)` to `b`, recursing structurally. /// -/// Safe to do after the main desugar walk: every remaining `e` is pure +/// Safe to do after the main channelize walk: every remaining `e` is pure /// (its `Feed`/`Define` sites have been extracted, leaving `Unit` /// residue), so dropping it is value-preserving. fn drop_expr_stmts(expr: Expr) -> Expr { @@ -1037,7 +1038,7 @@ fn drop_expr_stmts(expr: Expr) -> Expr { } /// Whether the subtree contains a pre-phase marker node (`For`/`MutWrite`) -/// that the unified letrec phase consumes downstream of desugar. Used to +/// that the unified letrec phase consumes downstream of channelize. Used to /// keep marker-bearing `ExprStmt`s alive through [`drop_expr_stmts`]. fn contains_phase_marker(expr: &Expr) -> bool { if matches!( @@ -1051,7 +1052,7 @@ fn contains_phase_marker(expr: &Expr) -> bool { found } -/// Confirm that no `Defer`/`Feed`/`Define` nodes remain after desugar. +/// Confirm that no `Defer`/`Feed`/`Define` nodes remain after channelize. fn assert_no_defer_residue(expr: &Expr) -> Result<(), DeferError> { match &expr.node { // `mut_elim::run` (before channelize) eliminates every mutable variable @@ -1139,13 +1140,13 @@ fn assert_no_defer_residue(expr: &Expr) -> Result<(), DeferError> { /// When found, processes the binding via [`channelize_defer`] (feed path) or /// inlines the define value directly (define path). All other nodes are /// recursed into structurally. -fn desugar(expr: Expr, ctx: &mut DesugarCtx) -> Result { - // One frame per node over the whole tree; grow on demand, as the other +fn channelize_expr(expr: Expr, ctx: &mut ChannelizeCtx) -> Result { + // One stack frame per node over the whole tree; grow on demand, as the other // pass-level walks do. - stacker::maybe_grow(512 * 1024, 1024 * 1024, || desugar_inner(expr, ctx)) + stacker::maybe_grow(512 * 1024, 1024 * 1024, || channelize_inner(expr, ctx)) } -fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { +fn channelize_inner(expr: Expr, ctx: &mut ChannelizeCtx) -> Result { if matches!(expr.node, TypedExprNode::Error) { crate::unexpected_error_node!(); } @@ -1214,7 +1215,12 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { // Recurse into the body first to handle any nested // non-clustered defers (inner `let d = Defer in ...` // separated from this cluster by other lets). - let body_rewritten = desugar(current_body, ctx)?; + let body_rewritten = channelize_expr(current_body, ctx)?; + // The **outermost** `let d = Defer` is the slot: the cluster's whole + // product replaces it. A cluster's inner defers are consumed too but + // are not named, because naming them would assert they die, and a + // defer whose handle survives in a type does not. + let _g = lineage::enter(node_id, "channelize.cluster", lineage::Nature::Expansion); channelize_cluster(&defer_names, &chan_names, body_rewritten, ctx) } TypedExprNode::Let { @@ -1237,7 +1243,23 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { // Read before the lift consumes the binding. let (outer, lvl) = handle_chan_dom(&binding.ty) .unwrap_or_else(|| (binding.name.clone(), crate::ccl::ChanLevel(0))); - let lift = lift_defer(&binding.name, *bound_expr, &body); + // The lift *mints*: `channelize_substitute` replaces the inner + // scope's trailing `Var` with the outer body, and any `ExprStmt` + // prefix is rebuilt onto the lifted spine. Those products stand in + // for this `let`, which the lift consumes, so it is the slot. + // `Machinery` — merging two defer scopes is plumbing that undoes an + // inlining artifact, not anything the user wrote. + // + // The recursion below runs outside the recording, so a nested lift + // attributes to its own `let`. + let lift = { + let _g = lineage::enter( + node_id, + "channelize.defer_lift", + lineage::Nature::Machinery, + ); + lift_defer(&binding.name, *bound_expr, &body) + }; // The lift renames the inner defer binder to the outer name, // but *consumer types outside the lifted subtree* may carry // the inner handle's rigid `ChanDom`. Record the alias so the @@ -1257,7 +1279,7 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { ctx.resolved_domains .push((inner_key, Type::ChanDom(outer, lvl))); } - return desugar(lift.expr, ctx); + return channelize_expr(lift.expr, ctx); } // Let-of-defer-returning-let collapse: `let y = (let z = // E in Var(z)) in body_y` is equivalent to `let z = E in @@ -1273,6 +1295,15 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { && is_defer_returning(inner_body, &inner_binding.name) && contains_defer(inner_be) { + // This arm rebuilds: it copies the inner scope out of the + // borrowed tree and splices the outer body into its tail, so the + // collapsed `let` and both copies are new nodes standing in for + // this `let`. Same slot and same reason as the lift above. + let _g = lineage::enter( + node_id, + "channelize.defer_collapse", + lineage::Nature::Machinery, + ); let inner_name = inner_binding.name.clone(); let inner_be = (**inner_be).clone(); let inner_body = (**inner_body).clone(); @@ -1283,7 +1314,7 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { // Rename inner_name → binding.name in the spliced body // so the inner defer is exposed under the outer let-y // name for subsequent passes. - let renamed = desugar_rename(spliced, &inner_name, &binding.name); + let renamed = channelize_rename(spliced, &inner_name, &binding.name); // Same alias recording as the lift above — types outside this // subtree may carry the inner handle's rigid name. let inner_key = handle_chan_dom(&inner_binding.ty) @@ -1296,12 +1327,13 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { .push((inner_key, Type::ChanDom(outer, lvl))); } let collapsed = Expr::let_bind(binding.name.clone(), inner_be, renamed); - return desugar(collapsed, ctx); + drop(_g); + return channelize_expr(collapsed, ctx); } // Recurse first so any inner aliases / UDF-inlines get // resolved before we check this outer binding. - let bound_expr = desugar(*bound_expr, ctx)?; - let body = desugar(*body, ctx)?; + let bound_expr = channelize_expr(*bound_expr, ctx)?; + let body = channelize_expr(*body, ctx)?; // Alias inlining (`let y = Var(x) in body` → `body[y → x]`) is // `inline`'s job, not channelize's: `inline` unconditionally collapses // a bare-`Var` alias before this pass runs — post-uniquify its @@ -1346,7 +1378,7 @@ fn desugar_inner(expr: Expr, ctx: &mut DesugarCtx) -> Result { user_annotation, node_id, }; - expr.try_map_children(|c| desugar(c, ctx))?; + expr.try_map_children(|c| channelize_expr(c, ctx))?; Ok(expr) } } @@ -1370,7 +1402,7 @@ fn channelize_cluster( defer_names: &[Name], chan_names: &HashMap, body: Expr, - ctx: &mut DesugarCtx, + ctx: &mut ChannelizeCtx, ) -> Result { // Extract feeds/defines for each defer. `rewritten` accumulates the // body's Feed/Define replacements as we process each defer in turn. @@ -2000,7 +2032,7 @@ fn compose_typed_or_hole(elts: Vec) -> Expr { /// /// `in_inner_scope` is `true` when the walk has crossed a [`TypedExprNode::Lambda`] /// or [`TypedExprNode::Case`] branch boundary — `Define` is disallowed in those -/// contexts since the desugared binding would need to escape the inner scope. +/// contexts since the channelized binding would need to escape the inner scope. fn extract_for_defer( expr: Expr, defer_name: &Name, @@ -2009,7 +2041,7 @@ fn extract_for_defer( in_inner_scope: bool, ) -> Result { // Grow the stack on demand, as `lambda_elim`'s two recursion entries do. This - // walk descends the whole tree in one frame per node, and the frame is large + // walk descends the whole tree in one stack frame per node, and the frame is large // (one `match` over every node kind, so it is sized for the union of all arms) // — deep enough trees overflow a test thread's default stack. Every level goes // through this wrapper, so each one checks the remaining headroom. @@ -2148,12 +2180,19 @@ fn extract_for_defer_impl( // stamp the wrap at construction — // the let's type is its body's, closed over the binder // (the design §6.2 discharge) — there is no - // re-derivation pass to fill a `Hole` in. (The discharge - // clone here only feeds `apply_type`; its nodes land in - // the type/predicate domain, so it is left un-freshened.) - let let_ty = - crate::ccl::subst::Subst::discharge(&binding.name, bound_expr.clone()) - .apply_type(&original.ty); + // re-derivation pass to fill a `Hole` in. The discharge + // payload only feeds `apply_type`, so it is a *template* + // rather than a tree node — it keeps its ids, and the + // sibling is minted at the read inside `apply_type`. + let let_ty = crate::ccl::subst::Subst::discharge( + &binding.name, + bound_expr.clone_preserving_ids(), + ) + .apply_type(&original.ty); + // The `Let` this walk is rebuilding keeps the original + // `bound_expr` in the body, and each extracted feed that + // captures the binder gets its own re-binding of the same + // definition, so every wrap is a copy. *feed = Expr::let_bind(binding.name.clone(), bound_expr.clone(), original) .with_ty(let_ty); } @@ -2797,11 +2836,11 @@ mod tests { let body = Expr::expr_stmt(Expr::feed("d", lit(1)), var("d")); let expr = Expr::let_bind("d", Expr::new(TypedExprNode::Defer), body); let result = run(typed(expr)).unwrap(); - // After desugar: let __scope_out_d_0 = (Unit; Record({result: d, to_d: 1})) in + // After channelize: let __scope_out_d_0 = (Unit; Record({result: d, to_d: 1})) in // let d = __scope_out_d_0.to_d in // __scope_out_d_0.result let s = symbolic(&result); - // After desugar: `unit; let d = (λ __unused → 1) in d` — the + // After channelize: `unit; let d = (λ __unused → 1) in d` — the // scalar feed value is lifted to `Fun(Unit, T)` via the // `λ __unused → V` wrap, then bound to the defer name. assert!(s.contains("__unused"), "expected const-wrap in output: {s}"); diff --git a/src/ccl/context.rs b/src/ccl/context.rs index 9326874d..f9c52333 100644 --- a/src/ccl/context.rs +++ b/src/ccl/context.rs @@ -14,13 +14,16 @@ use crate::{ Expr, channelize, infer::{ InferError, TypeInferenceContext, check_mut_discipline, check_mut_write_targets, - check_pre_desugar, infer, typecheck, + check_pre_channelize, infer, typecheck, }, inline, lambda_elim, - lineage::{Leak, RecorderSession, SourceProjection, collapse_lowering}, + lineage::{ + Leak, LineageMap, LineageTable, LoweringSession, PassScope, SourceProjection, + TableSession, collapse, collapse_lowering, + }, lower::{LoweringContext, LoweringError, lower_stmts}, mut_elim, planning, - provenance::NodeId, + provenance::{NodeId, Pass}, symbolic::{symbolic, symbolic_typed}, transact_phase, uniquify, }, @@ -68,15 +71,15 @@ pub enum CompileError { /// The (parseable) AST uses a construct the lowering pass does not /// support yet. Lower(LoweringError), - /// Defer/Feed/Define desugaring rejected the program (e.g. a defer + /// Defer/Feed/Define channelization rejected the program (e.g. a defer /// binding with no feeds, mixed `<<`/`<<=` on the same handle, or a /// `<<=` inside a non-top-level scope). /// - /// Desugaring runs *after* inference (so type errors report against + /// Channelization runs *after* inference (so type errors report against /// the user's program shape); a program with both a type error and a /// structural defer error therefore surfaces only the type error /// first. - DesugarDefers(channelize::DeferError), + ChannelizeDefers(channelize::DeferError), /// Type inference rejected one expression. /// /// `span` is the offending source range, resolved at the `compile_program` @@ -129,7 +132,7 @@ impl CompileError { .write((src_name, ariadne::Source::from(src)), &mut buf) .expect("ariadne write should not fail on Vec"); } - CompileError::DesugarDefers(e) => { + CompileError::ChannelizeDefers(e) => { buf.extend_from_slice(format!("error: deferred collection: {e}\n").as_bytes()); } CompileError::Infer { @@ -249,7 +252,7 @@ impl From for CompileError { impl From for CompileError { fn from(e: channelize::DeferError) -> Self { - Self::DesugarDefers(e) + Self::ChannelizeDefers(e) } } @@ -290,7 +293,7 @@ impl IntoCompileErrors for lambda_elim::LambdaElimError { impl IntoCompileErrors for channelize::DeferError { fn into_compile_errors(self) -> Vec { - vec![CompileError::DesugarDefers(self)] + vec![CompileError::ChannelizeDefers(self)] } } @@ -519,7 +522,7 @@ pub struct CompiledProgram { /// execution-shaped (point-free, fused) — the wrong tree for a source-level /// view. The inspector anchors here instead. pub post_inference_ir: Expr, - /// The post-desugar IR snapshot — the inspector's **downstream** pane, one + /// The post-channelize IR snapshot — the inspector's **downstream** pane, one /// pipeline stage *below* [`post_inference_ir`](Self::post_inference_ir). /// /// This is `expr` captured **right after `channelize`** (which now runs @@ -529,9 +532,35 @@ pub struct CompiledProgram { /// fan-ins) are present. Because monomorphization ran earlier (inside /// `infer`), this tree is post-mono like [`post_inference_ir`](Self::post_inference_ir). /// - /// Every id preserved through inline/transact/letrec/desugar is shared with + /// Every id preserved through inline/transact/letrec/channelize is shared with /// [`post_inference_ir`](Self::post_inference_ir). - pub post_desugar_ir: Expr, + pub post_channelize_ir: Expr, + /// The compile's lineage record: `NodeId → { parents, blame, rule }`, one + /// row per node a pass produced, written by the recorder as those passes run + /// ([`crate::ccl::lineage`]). + /// + /// One table covers the whole compile, because a row's key is a + /// process-unique `NodeId` and needs no pass set to disambiguate it; a row's + /// `via` is what a pane relation restricts by. The passes that record are + /// [`Pass::Mono`] (everything monomorphization mints inside `infer`, which + /// bridges the pre-inference ⇄ post-inference panes) and [`Pass::Inline`], + /// [`Pass::Transact`], [`Pass::Letrec`], [`Pass::Channelize`] (the four passes + /// between the post-inference and post-channelize snapshots) — see + /// [`MONO_PASSES`] and [`CHANNELIZE_PASSES`]. + /// + /// Rows exist only for nodes a recording produced: an untouched node has none + /// (it was never rewritten), and neither has a refinement-predicate interior + /// (see `LineageTable`'s `TODO(predicate-rows)`). + /// + /// Empty when capture is switched off — no pass scope is opened then, so + /// every flush is a no-op — see [`lineage_capture_enabled`]. This is the + /// authoritative lineage surface: + /// [`materialize_panes`](Self::materialize_panes) folds it for each pane + /// relation. + // Consumed by `materialize_panes` and the inspector model; the compiler + // itself never reads it. + #[allow(dead_code)] + pub(crate) lineage_table: LineageTable, /// The parsed CHL surface AST — the source-of-truth for source-level /// (lexical) inspector queries. /// @@ -574,6 +603,201 @@ impl CompiledProgram { pub fn sinks(&self) -> impl Iterator { self.outputs.iter().filter(|o| !o.is_main()) } + + /// Fold [`lineage_table`](Self::lineage_table) across the two pane relations into + /// the per-pane [`SourceProjection`]s, the pane-pair [`LineageMap`]s, and + /// each relation's [`Leak`]s. Cold path (snapshot-serve only), never called + /// by [`compile_program`]: + /// + /// * pre-inference pane = the lowering projection (`uniquify` preserves every + /// id in place, so lowering's keys are still the pane's keys); + /// * post-inference pane = fold the [`MONO_PASSES`] rows against the + /// pre-inference pane; + /// * post-channelize pane = fold the [`CHANNELIZE_PASSES`] rows against the + /// post-inference pane. + /// + /// The leaks are **returned, not asserted**: `Unexplained` is the capture + /// gate ([`gate_leaks`]) but a relation is only clean once every pass inside + /// it records its rewrites, and `Died` is the death report, which a caller + /// reads rather than gates on. + // Cold path: the inspector's snapshot serve, which is not in this workspace. + #[allow(dead_code)] + pub(crate) fn materialize_panes(&self) -> MaterializedPanes { + let pre_ids = collect_tree_ids(&self.pre_inference_ir); + let post_inf_ids = collect_tree_ids(&self.post_inference_ir); + let post_des_ids = collect_tree_ids(&self.post_channelize_ir); + + let (mono_map, post_inference, mono_leaks) = collapse( + &self.lineage_table, + MONO_PASSES, + &pre_ids, + &post_inf_ids, + &self.lowering_projection, + ); + let (channelize_map, post_channelize, channelize_leaks) = collapse( + &self.lineage_table, + CHANNELIZE_PASSES, + &post_inf_ids, + &post_des_ids, + &post_inference, + ); + + MaterializedPanes { + pre_inference: self.lowering_projection.clone(), + post_inference, + post_channelize, + mono_map, + channelize_map, + mono_leaks, + channelize_leaks, + } + } +} + +/// The passes each pane relation spans — the set +/// [`CompiledProgram::materialize_panes`] restricts the whole-compile table by. +/// +/// A pane relation is defined by the passes that ran between its panes, not by a +/// position in a list: a program that skips a pass must not shift the other one's +/// set, and a row produced outside a relation's passes has to read to it as an +/// ordinary un-produced id. +pub(crate) const MONO_PASSES: &[Pass] = &[Pass::Mono]; + +/// The passes between the post-inference and post-channelize panes. See +/// [`MONO_PASSES`]. +pub(crate) const CHANNELIZE_PASSES: &[Pass] = + &[Pass::Inline, Pass::Transact, Pass::Letrec, Pass::Channelize]; + +/// The per-pane projections, pane-pair lineage maps, and per-relation leaks +/// materialized from [`CompiledProgram::lineage_table`] — see +/// [`CompiledProgram::materialize_panes`]. +// Consumed by the inspector model; unused within the compiler itself. +#[allow(dead_code)] +pub(crate) struct MaterializedPanes { + /// pre-inference pane projection (= the lowering projection). + pub(crate) pre_inference: SourceProjection, + /// post-inference pane projection. + pub(crate) post_inference: SourceProjection, + /// post-channelize pane projection. + pub(crate) post_channelize: SourceProjection, + /// pre-inference → post-inference lineage map (the `Mono` fan-out). Dense: + /// an id that survived is its own self-edge. + pub(crate) mono_map: LineageMap, + /// post-inference → post-channelize lineage map. + pub(crate) channelize_map: LineageMap, + /// Leaks at the pre-inference → post-inference pane relation. + pub(crate) mono_leaks: Vec, + /// Leaks at the post-inference → post-channelize pane relation. + pub(crate) channelize_leaks: Vec, +} + +impl MaterializedPanes { + /// `(relation name, leaks)` for each pane relation, in pipeline order. + #[allow(dead_code)] + pub(crate) fn pane_relations(&self) -> [(&'static str, &[Leak]); 2] { + [ + ("pre-inference → post-inference", &self.mono_leaks), + ("post-inference → post-channelize", &self.channelize_leaks), + ] + } + + /// The pane relations a gate can hold at zero: the ones whose passes are + /// **instrumented**. Same discipline as an audit span's endpoint — a gate + /// over a relation whose passes do not record cannot reach zero however + /// correct the recording is, so it would report a constant rather than a + /// regression. + /// + /// Today that is the second relation only. The first spans monomorphization + /// and inference, and **inference's predicate producers do not record**: + /// `specialize_use` clones a definition per instantiation, and the copies of + /// a predicate term inside it row against interior ids no pass ever produced + /// (`Leak::ParentUnknown`). Over `tests/compilation_pipeline` that is 5 + /// programs, all of them UDF-with-filter or poly-wrapper shapes, and all of + /// it the crossing "Known prerequisites" records. **Add the first relation + /// here in the commit that makes inference record**, exactly as an audit + /// span's endpoint moves with the pass that earns it. + pub(crate) fn gated_pane_relations(&self) -> [(&'static str, &[Leak]); 1] { + [("post-inference → post-channelize", &self.channelize_leaks)] + } +} + +/// Ids duplicated across *distinct* predicate terms — the uniqueness question +/// `assert_unique_node_ids` does not answer, since it walks the main tree only. +/// +/// Dedups by `Rc` pointer first — one term riding many type slots is one term +/// and shares its ids with itself legitimately — then reports any id carried by +/// two different terms, or by a predicate term and the main tree. +#[cfg(test)] +pub(crate) fn predicate_id_collisions(expr: &Expr) -> Vec<(NodeId, &'static str)> { + use crate::ccl::ty::Type; + use std::collections::{HashMap, HashSet}; + use std::rc::Rc; + + // One entry per *distinct* predicate term, keyed by `Rc` pointer: a term + // riding many type slots is one term and shares its ids with itself. + fn ids_of(e: &Expr, out: &mut HashSet) { + out.insert(e.node_id()); + e.walk_children(|c| ids_of(c, out)); + } + fn from_ty(t: &Type, acc: &mut HashMap>) { + if let Type::Refinement(_, r) = t { + let key = Rc::as_ptr(&r.predicate) as usize; + if let std::collections::hash_map::Entry::Vacant(slot) = acc.entry(key) { + let mut s = HashSet::new(); + ids_of(&r.predicate, &mut s); + slot.insert(s); + } + from_expr_ty(&r.predicate, acc); + } + t.walk_children(|c| from_ty(c, acc)); + } + fn from_expr_ty(e: &Expr, acc: &mut HashMap>) { + from_ty(&e.ty, acc); + if let Some(a) = &e.user_annotation { + from_ty(a, acc); + } + if let crate::ccl::TypedExprNode::Cast { target, .. } = &e.node { + from_ty(target, acc); + } + e.walk_children(|c| from_expr_ty(c, acc)); + } + let mut terms: HashMap> = HashMap::new(); + from_expr_ty(expr, &mut terms); + + let main = collect_main_tree_ids(expr); + let mut seen: HashSet = HashSet::new(); + let mut dups = Vec::new(); + for local in terms.values() { + for id in local { + if main.contains(id) { + dups.push((*id, "predicate-vs-main-tree")); + } else if !seen.insert(*id) { + dups.push((*id, "predicate-vs-predicate")); + } + } + } + dups.sort(); + dups.dedup(); + dups +} + +/// Every [`NodeId`] on the **main tree only** — the `walk_children` node-set, +/// refinement predicates excluded. +/// +/// The counterpart to [`collect_tree_ids`], which is predicate-*inclusive* and is +/// the id domain the fold must explain. This narrow walk exists so the two can be +/// measured against the same logs (see [`LineageAudit::live_ids`]): with the +/// narrow live set, planning's *main-tree* output is essentially fully explained +/// and the residue is entirely inside refinement predicates, which is the +/// measurement that says where the remaining work is. +pub(crate) fn collect_main_tree_ids(expr: &Expr) -> std::collections::HashSet { + fn go(e: &Expr, acc: &mut std::collections::HashSet) { + acc.insert(e.node_id()); + e.walk_children(|c| go(c, acc)); + } + let mut acc = std::collections::HashSet::new(); + go(expr, &mut acc); + acc } /// Every node id reachable in `expr`: the `walk_children` node set plus the @@ -613,23 +837,36 @@ pub(crate) fn collect_tree_ids(expr: &Expr) -> std::collections::HashSet acc } -/// The lowering-boundary leak gate: the lowering log records every mint and -/// copy at its site, so [`collapse_lowering`]'s fold must explain every -/// output-tree node with **no** leak of any class — an `Unexplained` (an -/// unrecorded lowering mint) is a recording bug, not tolerated residue. +/// The pane-relation leak gate: **[`Leak::Unexplained`] and [`Leak::ParentUnknown`] +/// must be zero**. +/// +/// `Unexplained` is the capture gate — an output-pane node with no origin means +/// the driver missed a mint. `ParentUnknown` is the record-integrity gate — a +/// node's lineage stopping at an id the fold has never heard of. +/// +/// [`Leak::Died`] is deliberately **not** gated. Nothing declares a fate under +/// driver capture, so `Died` fires for every node that dies across the fold — it +/// is the death report, and asserting it empty would be unsatisfiable on any +/// program that rewrites anything. +/// /// Debug/test only, single code path (`cfg!`, not `#[cfg]`). -fn assert_leaks_clean(leaks: &[Leak], boundary: &str) { +fn gate_leaks(leaks: &[Leak], relation: &str) { if !cfg!(any(debug_assertions, test)) { return; } + let defects: Vec<&Leak> = leaks.iter().filter(|l| l.is_defect()).collect(); assert!( - leaks.is_empty(), - "lineage leak at the fully-recorded {boundary} boundary (expected none): {leaks:?}" + defects.is_empty(), + "lineage capture defect across the {relation} pane relation \ + ({} of {} leaks; `Died` is the death report and is not gated): {defects:?}", + defects.len(), + leaks.len(), ); } /// Every duplicated [`NodeId`] over the **main tree** — the `walk_children` -/// node-set, refinement predicates excluded (matching inline's blind spot, so the +/// node-set, refinement predicates excluded — uniqueness is a narrower question +/// than explanation, and this matches inline's blind spot, so the /// check does not false-fire there). Returns `(id, node kind)` for each /// occurrence *beyond the first*. fn duplicate_node_ids(expr: &Expr) -> Vec<(NodeId, &'static str)> { @@ -662,9 +899,13 @@ fn duplicate_node_ids(expr: &Expr) -> Vec<(NodeId, &'static str)> { /// It catches the *class* of preserve-as-mint / clone-without-freshen bugs /// across the whole test suite, not just a crafted program. /// -/// The walk is the same main-tree `walk_children` walk as -/// [`duplicate_node_ids`]/[`collect_tree_ids`] — a predicate-inclusive walk would -/// false-fire on inline's known predicate blind spot. Gated +/// The walk is [`duplicate_node_ids`]'s main-tree `walk_children` walk, and +/// **deliberately narrower than [`collect_tree_ids`]**, which enumerates +/// refinement predicates too. The two answer different questions — explanation +/// versus uniqueness (`design/provenance.md`, "Walking the ids") — and a +/// predicate-inclusive uniqueness walk would false-fire on inline's known +/// predicate blind spot, where a predicate interior legitimately aliases a +/// main-tree id. Gated /// via `cfg!(...)` as an expression (not a `#[cfg]` item) so the same call site /// compiles under both `./ci.sh` clippy passes without a release-only /// gated-item-reference failure. @@ -698,6 +939,209 @@ pub(crate) fn assert_unique_node_ids(expr: &Expr, boundary: &str) { ); } +// --------------------------------------------------------------------------- +// Lineage measurement switches +// +// Four environment variables steer what the recorder does. Each is named once +// and read through exactly one accessor, because two of them interact: pane +// capture and an audit span are alternative modes over *one* recorder session +// (per-thread, non-reentrant), so the two switches have to agree on what "an +// audit is running" means. A second `std::env::var` call on the same name is +// how that agreement silently drifts. +// --------------------------------------------------------------------------- + +/// Turns pane capture off (`=0`), so the cost of capture can be measured +/// against the same binary compiling the same programs. +const LINEAGE_ENV: &str = "CAMBRA_LINEAGE"; + +/// Names the [`LineageAudit`] span to open, e.g. `full`. +const LINEAGE_AUDIT_ENV: &str = "CAMBRA_LINEAGE_AUDIT"; + +/// Gates the pane-relation leak classes on **every** compile (`=1`), making the +/// gate's corpus whatever the caller compiles. +const LINEAGE_GATE_ENV: &str = "CAMBRA_LINEAGE_GATE"; + +/// Narrows an audit's live set to the main tree, excluding refinement-predicate +/// interiors (`=0`). Predicate-inclusive otherwise — see +/// [`lineage_predicates_live`]. +const LINEAGE_PREDICATES_ENV: &str = "CAMBRA_LINEAGE_PREDICATES"; + +/// Repetition count for the ignored perf driver. Test-only — the driver is a +/// `#[test]`, so the name is dead in a lib build; it lives here rather than +/// beside its reader so that adding a fifth switch means editing one block. +#[cfg(test)] +const PERF_REPS_ENV: &str = "CAMBRA_PERF_REPS"; + +/// The audit span named by [`LINEAGE_AUDIT_ENV`], if any. **Sole reader** of +/// that variable — see the section note above. +fn lineage_audit_span() -> Option { + std::env::var(LINEAGE_AUDIT_ENV).ok() +} + +/// Whether an audit's live set admits refinement-predicate interiors. **On by +/// default**; `=0` narrows it to [`collect_main_tree_ids`]'s `walk_children` +/// domain. +/// +/// The default matches what the shipped gate folds: [`materialize_panes`] takes +/// every pane's live set with [`collect_tree_ids`], which is predicate-inclusive. +/// An audit measuring the *narrower* set therefore reports edges the gate does +/// not — a recorded predicate rebuild's parents are input-tree predicate +/// interiors, which the narrow set omits from the input side, so those edges +/// dangle as [`Leak::ParentUnknown`] with nothing actually unrecorded. Defaulting +/// to the narrow set made the plain invocation report ~166 folds of noise on +/// the pipeline corpus and buried real defects in it. +/// +/// [`materialize_panes`]: CompiledProgram::materialize_panes +fn lineage_predicates_live() -> bool { + !std::env::var(LINEAGE_PREDICATES_ENV).is_ok_and(|v| v == "0") +} + +/// Whether [`compile_program`] opens the per-pass recorder scopes that fill +/// [`CompiledProgram::lineage_table`]. On by default. +/// +/// The switch changes only whether a scope is opened; every recording hook is +/// already a no-op outside one (`STEP_STACK` empty / no ambient pass). +/// +/// A [`LineageAudit`] span opens its own scope, so naming one turns pane +/// capture off. +pub(crate) fn lineage_capture_enabled() -> bool { + !std::env::var(LINEAGE_ENV).is_ok_and(|v| v == "0") && lineage_audit_span().is_none() +} + +/// Whether every compile folds its pane relations and gates the leak classes, +/// rather than only the programs a test asks about. +/// +/// **What this buys.** The always-on gate is +/// `pane_relations_fold_with_no_structural_leaks`, whose corpus is the handful +/// of programs listed in `corpus()`. That is a *sample*, and a recording gap in +/// a shape the sample misses is invisible: the unrecorded +/// `fold_induction_loop` call in `transact_phase` (a commit decision reading +/// another loop's accumulator) and `flatten_spine`'s value-position writer hoist +/// both sat outside it. With this on, the corpus is every program the caller +/// compiles — point it at `tests/compilation_pipeline` and the gate covers the +/// whole suite instead of eleven programs. +/// +/// Off by default because it folds the table twice per compile, which is +/// superlinear work the ordinary pipeline does not need; CI turns it on. It is +/// **not** a substitute for the sampled gate, which stays always-on so a plain +/// `cargo test` still fails on the common shapes. +/// +/// Requires capture: with `CAMBRA_LINEAGE=0`, or under an audit span (which +/// takes the pass scopes for itself), the table is empty and every output node +/// would read as unexplained. Both are honoured rather than asserted, so a run +/// can name one without also having to unset this. +fn lineage_gate_every_compile() -> bool { + std::env::var(LINEAGE_GATE_ENV).is_ok_and(|v| v != "0") && lineage_capture_enabled() +} + +/// Run `f` with `pass` installed as the recorder's ambient pass, so every row a +/// recording inside it writes is tagged with that pass. With `capture` false this +/// is exactly `f()` — no scope, and every construction hook stays a no-op. +/// +/// The pass identity lives in the *data* — the row's `RewriteTag`, completed +/// from the scope when a guard drops — which is why one helper covers every pass +/// regardless of its signature: the passes here are free functions of four +/// different shapes and nothing is threaded through them. +fn recorded(capture: bool, pass: Pass, f: impl FnOnce() -> R) -> R { + if !capture { + return f(); + } + let _scope = PassScope::enter(pass); + f() +} + +/// A driver-capture audit over one span of the pipeline: install a pass recorder at +/// the input pane, fold at the output pane, and print what the capture explains. +/// +/// Opt-in via `CAMBRA_LINEAGE_AUDIT=1` so the whole test suite can run either +/// way. It is a **measurement**, not a gate: the point of driver capture is that +/// nothing declares a fate, so the fold's `retired` set is empty by construction +/// and every genuinely-dead input id surfaces as [`Leak::Died`]. That class is +/// therefore the *death report*, not an error, and the audit counts it +/// separately from the classes that really are recording bugs +/// ([`Leak::Unexplained`] — an output node no recording accounted for). +struct LineageAudit { + span: &'static str, + state: Option<(PassScope, std::collections::HashSet)>, +} + +impl LineageAudit { + /// The `via` an audit's rows carry. A span covers several passes under one + /// scope, so no single pass is the truthful answer; the tag is nominal, and + /// naming it once keeps the scope's tag and the passes the fold restricts by + /// from disagreeing about which nominal pass it is. + const AUDIT_VIA: Pass = Pass::Planning; + + /// The audit's live set: [`collect_main_tree_ids`]'s `walk_children` domain, + /// or — when [`lineage_predicates_live`] says so — [`collect_tree_ids`]'s + /// predicate-inclusive domain. + /// + /// Both arms are real and differ: the narrow one is *not* the id domain any + /// more. `collect_tree_ids` became predicate-inclusive, which briefly made + /// this switch inert (both arms computing the same set) and its own doc + /// comment false. Keeping a named narrow walk is what makes the comparison + /// measurable rather than a no-op. + fn live_ids(expr: &Expr) -> std::collections::HashSet { + if lineage_predicates_live() { + collect_tree_ids(expr) + } else { + collect_main_tree_ids(expr) + } + } + + /// Open the audit if [`lineage_audit_span`] names this `span`. Spans are + /// mutually exclusive because a pass scope is per-thread and non-reentrant; + /// naming them lets a run narrow the measurement to the passes under study + /// instead of the whole tail of the pipeline. + fn start(span: &str, name: &'static str, input: &Expr) -> Self { + let on = lineage_audit_span().is_some_and(|w| w == span); + LineageAudit { + span: name, + state: on.then(|| (PassScope::enter(Self::AUDIT_VIA), Self::live_ids(input))), + } + } + + fn finish(self, output: &Expr) { + let Some((scope, input_ids)) = self.state else { + return; + }; + // Close the scope before folding: the rows are in the compile's table, + // which is still installed (it is drained at the end of the compile), + // so the measurement reads them in place rather than draining anything. + drop(scope); + let output_ids = Self::live_ids(output); + let Some((rows, projection, leaks)) = crate::ccl::lineage::with_active_table(|table| { + let (_map, projection, leaks) = collapse( + table, + &[Self::AUDIT_VIA], + &input_ids, + &output_ids, + &SourceProjection::new(), + ); + (table.len(), projection, leaks) + }) else { + return; + }; + let mut counts = std::collections::BTreeMap::<&str, usize>::new(); + for l in &leaks { + *counts + .entry(match l { + Leak::Unexplained { .. } => "Unexplained(output not covered by any recording)", + Leak::Died { .. } => "Died(= the death report)", + Leak::ParentUnknown { .. } => "ParentUnknown", + }) + .or_default() += 1; + } + eprintln!( + "[lineage-audit {}] rows={rows} input={} output={} attributed={} leaks={counts:?}", + self.span, + input_ids.len(), + output_ids.len(), + projection.len(), + ); + } +} + /// Compile a CHL program and return its operator graph plus subscribed outputs. /// /// Returns a [`CompiledProgram`] whose `outputs` vector contains one entry @@ -731,6 +1175,13 @@ pub fn compile_program( // wrong, not the user's input. let mut errors: Vec = Vec::new(); + // The node table spans the **whole compile**, not a pass: its rows are keyed + // by `NodeId`, which is unique for the life of the process, so every pass + // session's flushes mirror into one table with no possibility of collision. + // Installed before the first session opens (the lowering one) and drained + // below; every early return drops it, clearing the slot. + let table_session = TableSession::install(); + let parse_result = chl_parser::parse_module(code); errors.extend(parse_result.errors.into_iter().map(CompileError::Parse)); let Some(module) = parse_result.value else { @@ -751,10 +1202,10 @@ pub fn compile_program( // The always-on lowering session: installed in every // build for the whole of lowering. Its leaf entries (`tag_source`/ - // `tag_machinery`) and copy-frame flushes (uncurry, compare-chain) record a + // `tag_machinery`) and copy-sink writes (uncurry, compare-chain) record a // `LoweringLog`, folded once at the handoff below into the always-on lowering // projection. It must fully drain before the first pass (Mono) session opens. - let lowering_session = RecorderSession::lowering(); + let lowering_session = LoweringSession::install(); let lower_result = lower_stmts(&module.body, ctx.lowering_ctx()); errors.extend(lower_result.errors.into_iter().map(CompileError::Lower)); let Some(mut expr) = lower_result.value else { @@ -782,7 +1233,7 @@ pub fn compile_program( // The fold's leak taxonomy enforces mint coverage: an unrecorded lowering // mint surfaces as `Leak::Unexplained` (every output-tree node must be // explained by a leaf or a copy). The checks are debug/test - // gated at the boundary via `assert_leaks_clean`; the fold itself is + // gated at the boundary via `gate_leaks`; the fold itself is // always-on (its product is release-critical). // Id uniqueness is the precondition for keying anything by `NodeId`, so gate // it before the fold that does exactly that: a duplicate here would silently @@ -791,21 +1242,21 @@ pub fn compile_program( // freshens) are what make this a live risk at this boundary. assert_unique_node_ids(&expr, "post-lowering"); - let lowering_log = lowering_session.into_lowering_log(); + let lowering_log = lowering_session.into_log(); let lowering_projection = { let output_ids = collect_tree_ids(&expr); let (projection, leaks) = collapse_lowering(&lowering_log, &output_ids); - assert_leaks_clean(&leaks, "lowering"); + gate_leaks(&leaks, "lowering"); projection }; - debug!("Lowered (pre-desugar):\n{}", symbolic(&expr)); + debug!("Lowered (pre-channelize):\n{}", symbolic(&expr)); // α-uniquify all binders (Barendregt convention): every binding site gets // a globally fresh `Name` uid, so shadowing ceases to exist before any - // pass that compares names. Must run before defer desugaring — desugar's + // pass that compares names. Must run before channelization — channelize's // rewrites splice and rename terms under the assumption that distinct - // binders are distinct names. (Desugar now runs after inference; see below.) + // binders are distinct names. (Channelize now runs after inference; see below.) expr = uniquify::run(expr); // Retain the pre-inference IR for the inspector's upstream pane before @@ -813,12 +1264,17 @@ pub fn compile_program( // still-hole-typed tree. Its ids resolve against the `lowering_projection` // (the pre-mono originals). See `CompiledProgram::pre_inference_ir`. // A pane snapshot: the same nodes as the live tree, observed at a point in - // time, so it preserves ids. A freshening clone would hand the boundary a + // time, so it preserves ids. A freshening clone would hand the pane a // structurally identical program sharing no identity with the one it is meant // to snapshot, and these ids must resolve against the `lowering_projection`, // which is keyed by the originals. let pre_inference_ir = expr.clone_preserving_ids(); + // Pass recording for the rows the two pane relations fold. One scope per + // pass, opened and closed in place: a pass scope is per-thread and + // non-reentrant, so the scopes are sequential, never nested. + let capture_lineage = lineage_capture_enabled(); + // Register every source (pre-registered + discovered during lowering) with // inference and operator-conversion now that the full source set is known. for (_name, source) in ctx.lowering_ctx().take_sources() { @@ -836,7 +1292,12 @@ pub fn compile_program( // Inference runs on the user-shaped tree — before channelize — so type // errors are reported against the program the user wrote, not the // channelized rewrite. - let infer_outcome = infer(&mut expr, ctx.inference_ctx()); + // Monomorphization runs inside `infer` and is the only thing there that + // mints or clones nodes, so this session *is* the pre-inference → + // post-inference pane bridge. + let infer_outcome = recorded(capture_lineage, Pass::Mono, || { + infer(&mut expr, ctx.inference_ctx()) + }); // On failure, resolve each error's own blame node to a source span *here* — // the lowering projection is in scope and holds the lowered attribution (this // is the always-on release read: one hop, no fold). Every error names a node; @@ -859,14 +1320,14 @@ pub fn compile_program( debug!("Inferred:\n{}", symbolic(&expr)); debug!("Inferred (typed):\n{}", symbolic_typed(&expr)); // Consistency wall between `infer` and `channelize`. It is the relaxed - // *pre-desugar* check (`check_pre_desugar`), which permits the transient - // `Feed` / `Infer`-channel-domain types only desugar can erase. A failure + // *pre-channelize* check (`check_pre_channelize`), which permits the transient + // `Feed` / `Infer`-channel-domain types only channelize can erase. A failure // here is a compiler bug — with one exception: residual `Type::Infer` // variables, which inference deliberately tolerates for a generalized // definition the program never exercises at a concrete type (see // `Type::Infer`'s invariant). That residue is an *ambiguous program* — a // user error — so it is rendered as a diagnostic; anything else panics. - check_pre_desugar(&expr).map_err(|errs| { + check_pre_channelize(&expr).map_err(|errs| { if errs .iter() .all(|e| matches!(e, InferError::UnresolvedInfer { .. })) @@ -882,7 +1343,7 @@ pub fn compile_program( // fully-typed, still-`Mut`-bearing tree — after the consistency wall, // before inlining. It needs the pre-inline `Apply`/parameter structure // (rule 1's argument check) and the coalesced `.ty` slots and - // `user_annotation`s. Unlike the surrounding `check_pre_desugar` walls + // `user_annotation`s. Unlike the surrounding `check_pre_channelize` walls // (compiler-bug backstops), these are user errors: aliasing or nesting a // mutable reference. check_mut_discipline(&expr).map_err(|errs| errs.into_compile_errors())?; @@ -897,28 +1358,56 @@ pub fn compile_program( // binding — or to one monomorphization has since dropped. check_mut_write_targets(&expr).map_err(|errs| errs.into_compile_errors())?; - // Inline UDFs *before* desugar: a defer-mediating UDF (`λ out → out << e`) + // Inline UDFs *before* channelize: a defer-mediating UDF (`λ out → out << e`) // or a cross-function writer is beta-reduced to its call site before - // desugar routes feeds and before the unified letrec phase folds writers, + // channelize routes feeds and before the unified letrec phase folds writers, // both of which need their targets lexically present. Inlining runs on the // still-defer-bearing tree (Defer/Feed nodes and `Feed` types present) via // the defer-aware `Subst` engine (which renames a fed-to handle on // beta-reduction) and preserves defer-returning generators, so the - // post-inline wall is the relaxed `check_pre_desugar`, not strict + // post-inline wall is the relaxed `check_pre_channelize`, not strict // `typecheck`. // Retain the post-inference IR for the inspector before `inline` consumes // `expr`. This is the source-shaped, fully-typed anchor (lambdas intact, not - // yet point-free; inline/transact/letrec/desugar/lambda_elim/planning have + // yet point-free; inline/transact/letrec/channelize/lambda_elim/planning have // not run). `ast` (`join_planned`) is the *wrong* tree for a source // view — `lambda_elim`/`planning` re-mint ids and produce execution shape. // See `CompiledProgram::post_inference_ir`. // A pane snapshot; see `pre_inference_ir`. let post_inference_ir = expr.clone_preserving_ids(); - expr = inline::inline_capability_lambdas(expr); + // Driver-capture audit span: post-inference pane in, and out at the **last + // instrumented pane** — currently `post-as-of-read`, covering inline, + // transact, mut_elim, channelize and the as-of-read rewrite. + // + // The endpoint is chosen rather than inherited. An audit measures what the + // recordings explain, so a span running past the last instrumented pass + // reports every node the uninstrumented tail mints as a defect — a number + // that cannot reach zero however correct the recording is, which makes the + // audit read as a broken gate instead of a measurement. `lambda_elim` is the + // next pass here and records nothing (it re-mints nearly every pass-through + // node), so the span stops in front of it. + // + // **Move this endpoint when a pass becomes instrumented, in the same commit + // that instruments it** — to `post-lambda-elim` when the elim pass records, + // and to `join-planned` when planning does. Leaving it behind understates + // coverage; moving it ahead reintroduces the unreachable-zero problem. + let audit = LineageAudit::start( + "full", + "post-inference..post-as-of-read", + &post_inference_ir, + ); + // A narrower pane pair over just the mutability phases (inline, transact, + // mut_elim), which is where the fate-prediction question lives. + let audit_letrec = + LineageAudit::start("letrec", "post-inference..post-letrec", &post_inference_ir); + + expr = recorded(capture_lineage, Pass::Inline, || { + inline::inline_capability_lambdas(expr) + }); assert_unique_node_ids(&expr, "post-inline"); debug!("UDFs inlined CCL:\n{}", symbolic(&expr)); - check_pre_desugar(&expr).map_err(|errs| { + check_pre_channelize(&expr).map_err(|errs| { if errs .iter() .all(|e| matches!(e, InferError::UnresolvedInfer { .. })) @@ -968,24 +1457,31 @@ pub fn compile_program( // that same store, is decided per store and so lives inside the phase.) transact_phase::check_await_final_linearity(&expr) .map_err(|msg| vec![CompileError::Unsupported(msg)])?; - expr = transact_phase::run(expr, &txn_mut_vars) - .map_err(|msg| vec![CompileError::Unsupported(msg)])?; + expr = recorded(capture_lineage, Pass::Transact, || { + transact_phase::run(expr, &txn_mut_vars) + }) + .map_err(|msg| vec![CompileError::Unsupported(msg)])?; assert_unique_node_ids(&expr, "post-transact"); debug!("Transact phase CCL:\n{}", symbolic(&expr)); - check_pre_desugar(&expr).expect("transact phase produced an inconsistent tree"); + check_pre_channelize(&expr).expect("transact phase produced an inconsistent tree"); // The unified letrec phase: direct-mirror mutation loops (`For` / // `MutWrite`) become causal `LetRec` groups — mutable histories over // the induction domain, per src/ccl/design/mutability.md. Runs after // inlining (so cross-function writers land at their call sites) and // *before* channelize, so a per-iteration feed inside a loop is - // hoisted to an ordinary feed of the loop's history for desugar to route. + // hoisted to an ordinary feed of the loop's history for channelize to route. // The tree still carries Defer/Feed here, so the walls are the relaxed - // pre-desugar check. - let phase_out = mut_elim::run(expr); + // pre-channelize check. + // Isolated pane pair over `mut_elim` alone — the pass whose fate prediction + // driver capture is meant to delete. + let audit_mutelim = LineageAudit::start("mutelim", "post-transact..post-letrec", &expr); + let phase_out = recorded(capture_lineage, Pass::Letrec, || mut_elim::run(expr)); + audit_mutelim.finish(&phase_out); assert_unique_node_ids(&phase_out, "post-letrec-run"); + audit_letrec.finish(&phase_out); debug!("Letrec phase CCL:\n{}", symbolic(&phase_out)); - check_pre_desugar(&phase_out).expect("letrec phase produced an inconsistent tree"); + check_pre_channelize(&phase_out).expect("letrec phase produced an inconsistent tree"); // Feed channelization — the feed-routing step of the unified phase, run on // the phase-emitted `LetRec` tree (recognition happens *after* @@ -997,17 +1493,20 @@ pub fn compile_program( // remain: channelization is type-preserving by construction and closes // channel domains by substitution; the strict `typecheck` below is the // release-visible enforcement. - let mut desugared = channelize::run(phase_out).errs()?; - assert_unique_node_ids(&desugared, "post-desugar"); - debug!("Channelized:\n{}", symbolic(&desugared)); - typecheck(&desugared).expect("channelize produced an ill-typed tree"); + let mut channelized = recorded(capture_lineage, Pass::Channelize, || { + channelize::run(phase_out) + }) + .errs()?; + assert_unique_node_ids(&channelized, "post-channelize"); + debug!("Channelized:\n{}", symbolic(&channelized)); + typecheck(&channelized).expect("channelize produced an ill-typed tree"); // Retain the post-channelize tree for the inspector's downstream pane. On the - // post-inference desugar order this snapshot is *downstream* of + // post-inference channelize order this snapshot is *downstream* of // `post_inference_ir` (post-inline/transact/letrec/channelize); see the doc - // comment on `post_desugar_ir`. + // comment on `post_channelize_ir`. // A pane snapshot; see `pre_inference_ir`. - let post_desugar_ir = desugared.clone_preserving_ids(); + let post_channelize_ir = channelized.clone_preserving_ids(); // Fed-out mutable variable reads: rewrite a read-only reply that reads a mutable variable out of // its block into an outer-indexed as-of join (an as-of read at the reading @@ -1016,12 +1515,14 @@ pub fn compile_program( // rather than a point-free `const` a planning-time recognizer would have to // reject. Uniform across the reading loop's domain. See // `transact_phase::rewrite_as_of_reads`. - transact_phase::rewrite_as_of_reads(&mut desugared) + transact_phase::rewrite_as_of_reads(&mut channelized) .map_err(|msg| vec![CompileError::Unsupported(msg)])?; - assert_unique_node_ids(&desugared, "post-as-of-read"); - typecheck(&desugared).expect("as-of-read rewrite produced an ill-typed tree"); + assert_unique_node_ids(&channelized, "post-as-of-read"); + typecheck(&channelized).expect("as-of-read rewrite produced an ill-typed tree"); + // The last instrumented pane: see the span's own note at `LineageAudit::start`. + audit.finish(&channelized); - let lambda_elim = lambda_elim::run(desugared).errs()?; + let lambda_elim = lambda_elim::run(channelized).errs()?; assert_unique_node_ids(&lambda_elim, "post-lambda-elim"); debug!("λ-eliminated CCL:\n{}", symbolic(&lambda_elim)); debug!("λ-eliminated typed CCL:\n{}", symbolic_typed(&lambda_elim)); @@ -1042,7 +1543,11 @@ pub fn compile_program( debug!("Letrec recognized CCL:\n{}", symbolic(&recognized)); typecheck(&recognized).expect("letrec recognition produced an ill-typed tree"); + // Isolated pane pair over `planning::run` — the passes containing + // `simplify`'s 13 rules and `wrap_with_iterate`. + let audit_planning = LineageAudit::start("planning", "recognized..join-planned", &recognized); let join_planned = planning::run(recognized); + audit_planning.finish(&join_planned); assert_unique_node_ids(&join_planned, "post-planning"); debug!( "Join-planned CCL:\n{} : {}", @@ -1060,6 +1565,7 @@ pub fn compile_program( // equality, so the staging shapes now validate without re-blinding the // check or peeling cast refinements. typecheck(&join_planned).expect("type error after join planning"); + // Invariant (debug): planning's `iterate`/`restrict` markers live in the // term tree, never inside a type's refinement predicates — the substitution // boundary strips the neutral `iterate` marker (`ccl_utils::strip_iterate_markers`), @@ -1143,17 +1649,30 @@ pub fn compile_program( } } - Ok(CompiledProgram { + let lineage_table = table_session.into_table(); + + let program = CompiledProgram { ast: join_planned, outputs, done: done_rx, lowering_projection, pre_inference_ir, post_inference_ir, - post_desugar_ir, + post_channelize_ir, + lineage_table, source_ast: module, source: code.to_string(), - }) + }; + + // Every compile its own gate — see `lineage_gate_every_compile` for why this + // is opt-in and what it covers that the sampled gate does not. + if lineage_gate_every_compile() { + for (relation, leaks) in program.materialize_panes().gated_pane_relations() { + gate_leaks(leaks, relation); + } + } + + Ok(program) } #[cfg(test)] @@ -1161,6 +1680,657 @@ mod tests { use super::*; use rstest::rstest; + /// The pane-measurement corpus: every demo-gallery program that compiles + /// today, plus four inline programs covering the passes the gallery does not + /// reach (a `with begin():` transaction, a group-by, a UDF chain, and a + /// nested comprehension). + /// + /// The gallery's remaining programs are excluded for reasons unrelated to + /// lineage: most are deliberate *failure* fixtures (`while`, record-term + /// syntax, `Feed(_)` types) that pin errors and so have no panes to fold, + /// and the three HTTP demos bind a real listening socket during lowering, + /// which collides with itself under a parallel test runner. + fn corpus() -> Vec<(&'static str, String)> { + vec![ + ( + "arithmetic", + include_str!("../../tests/programs/arithmetic/program.cambra").to_string(), + ), + ( + "filter_and_aggregate", + include_str!("../../tests/programs/filter_and_aggregate/program.cambra").to_string(), + ), + ( + "for_accumulator", + include_str!("../../tests/programs/for_accumulator/program.cambra").to_string(), + ), + ( + "generator_pipeline", + include_str!("../../tests/programs/generator_pipeline/program.cambra").to_string(), + ), + ( + "inner_join", + include_str!("../../tests/programs/inner_join/program.cambra").to_string(), + ), + ( + "prefix_lines", + include_str!("../../tests/programs/prefix_lines/program.cambra").to_string(), + ), + ( + "streaming_echo", + include_str!("../../tests/programs/streaming_echo/program.cambra").to_string(), + ), + ( + "transaction", + "out = defer()\n\ + pool: Mut(Int, Txn) := 100\n\ + for r in [10, 20, 30]:\n\ + \x20 with begin():\n\ + \x20 pool := pool - r\n\ + with begin():\n\ + \x20 out << pool\n\ + out\n" + .to_string(), + ), + ( + "group_by", + "[sum(x) for x in groupby([y + 10 for y in [2,3,4,5,6] if y < 6], \\x -> x // 2)]\n" + .to_string(), + ), + ( + "udf_chain", + "def double(x):\n x * 2\ndef bump(x):\n double(x) + 1\n\ + xs = [1, 2, 3]\n[bump(x) for x in xs]\n" + .to_string(), + ), + ( + "feed_loop", + "out = defer()\nfor x in [1, 2, 3]:\n out << x * 2\nout\n".to_string(), + ), + ] + } + + /// Compile `code` through the full pipeline, panicking on error. + fn compile_ok(code: &str) -> CompiledProgram { + let mut ctx = GlobalContext::default(); + let consumer: Box = Box::new(|| {}); + match compile_program(&mut ctx, code, consumer) { + Ok(p) => p, + Err(errs) => panic!("expected a successful compile, got {errs:?}"), + } + } + + /// Per-leak-class counts, for reporting a pane relation. + #[derive(Default, Debug, PartialEq, Eq)] + struct LeakCounts { + unexplained: usize, + died: usize, + parent_unknown: usize, + } + + impl LeakCounts { + fn tally(leaks: &[Leak]) -> Self { + let mut c = LeakCounts::default(); + for l in leaks { + match l { + Leak::Unexplained { .. } => c.unexplained += 1, + Leak::Died { .. } => c.died += 1, + Leak::ParentUnknown { .. } => c.parent_unknown += 1, + } + } + c + } + + fn add(&mut self, o: &LeakCounts) { + self.unexplained += o.unexplained; + self.died += o.died; + self.parent_unknown += o.parent_unknown; + } + + /// The structural class — a record-integrity defect, independent of how + /// much of the pipeline records its rewrites. See [`gate_leaks`]. + fn structural(&self) -> usize { + self.parent_unknown + } + } + + /// **Capture totality**, as a corpus-wide property rather than a per-pass + /// assertion: both pane relations materialize and fold over every corpus + /// program, every output-pane node has an origin (`Unexplained == 0`), and + /// the structural classes are zero everywhere. + /// + /// The two invariants fail differently and are worth reading apart. + /// `structural == 0` says no node's lineage stops at an id the relation + /// never heard of. `Unexplained == 0` says no rewrite went + /// *unrecorded* — it is the gate the whole driver-capture design exists to + /// pass, and the number a newly-added rewrite site breaks first. + #[test] + fn pane_relations_fold_with_no_structural_leaks() { + let mut totals = [LeakCounts::default(), LeakCounts::default()]; + for (name, code) in corpus() { + let program = compile_ok(&code); + let panes = program.materialize_panes(); + for (i, (relation, leaks)) in panes.pane_relations().into_iter().enumerate() { + let c = LeakCounts::tally(leaks); + assert_eq!( + c.structural(), + 0, + "{name}: structural leaks across the {relation} pane relation: {c:?}" + ); + assert_eq!( + c.unexplained, 0, + "{name}: unexplained output nodes across the {relation} pane relation — a rewrite \ + that mints with nothing recording: {c:?}" + ); + eprintln!("[pane {name} / {relation}] {c:?}"); + totals[i].add(&c); + } + // The panes are the thing being materialized: each projection must + // hold entries for the tree it describes, and each map edges. + assert!(!panes.pre_inference.is_empty(), "{name}"); + assert!(!panes.post_inference.is_empty(), "{name}"); + assert!(!panes.post_channelize.is_empty(), "{name}"); + assert!(!panes.mono_map.edges().is_empty(), "{name}"); + assert!(!panes.channelize_map.edges().is_empty(), "{name}"); + } + eprintln!("[pane totals] pre→post-inference {:?}", totals[0]); + eprintln!( + "[pane totals] post-inference→post-channelize {:?}", + totals[1] + ); + } + + /// Every pass that records rows in a normal compile belongs to exactly one + /// pane relation, so no rewrite is folded twice and none is silently dropped. + /// + /// [`MONO_PASSES`] and [`CHANNELIZE_PASSES`] are the only things that decide + /// which pane relation a row reaches, so a pass that opens a scope without + /// joining neither would record rows nothing ever folds. + #[test] + fn every_recorded_pass_belongs_to_exactly_one_pane_relation() { + for (name, code) in corpus() { + let program = compile_ok(&code); + for p in program.lineage_table.recorded_passes() { + assert!( + MONO_PASSES.contains(&p) != CHANNELIZE_PASSES.contains(&p), + "{name}: {p:?} is in neither pane relation, or in both", + ); + } + } + } + + /// The `program / relation` pairs at which the pane relation is **not** + /// vacuous — where some node's origin is another node, so the fold had to + /// read a row. See + /// [`the_pane_folds_derive_a_non_vacuous_lineage_relation`]. + /// Grew from seven entries to sixteen — a strict superset, every original + /// retained — when monomorphization and transport-mode substitution started + /// recording. `pre-inference → post-inference` in particular was vacuous on + /// eight programs because the only pass rewriting there, `Mono`, + /// opened its recording *after* the clone that produced its nodes, so + /// nothing captured. + const EXERCISED_BOUNDARIES: &[&str] = &[ + "arithmetic / pre-inference → post-inference", + "feed_loop / post-inference → post-channelize", + "feed_loop / pre-inference → post-inference", + "filter_and_aggregate / pre-inference → post-inference", + "for_accumulator / post-inference → post-channelize", + "for_accumulator / pre-inference → post-inference", + "generator_pipeline / post-inference → post-channelize", + "generator_pipeline / pre-inference → post-inference", + "group_by / pre-inference → post-inference", + "inner_join / pre-inference → post-inference", + "prefix_lines / pre-inference → post-inference", + "streaming_echo / pre-inference → post-inference", + "transaction / post-inference → post-channelize", + "transaction / pre-inference → post-inference", + "udf_chain / post-inference → post-channelize", + "udf_chain / pre-inference → post-inference", + ]; + + /// **The pane relation is well-formed and non-vacuous** on every corpus + /// program: every edge runs from an input-pane id to an output-pane id, + /// every id present in both panes is its own dense self-edge, and the + /// pane relations where the fold had to read a *row* are exactly the ones + /// pinned above. + /// + /// The non-vacuity half is what this test is for. Fifteen of the + /// twenty-two corpus relations are pure identity: no pass inside them + /// minted, so the relation is the input pane's self-edges and holds for + /// reasons that have nothing to do with the recording. A corpus edit that + /// silently dropped the rewriting programs would otherwise leave a green + /// tautology behind. + #[test] + fn the_pane_folds_derive_a_non_vacuous_lineage_relation() { + let mut exercised: Vec = Vec::new(); + for (name, code) in corpus() { + let program = compile_ok(&code); + let panes = program.materialize_panes(); + let pre = collect_tree_ids(&program.pre_inference_ir); + let post_inf = collect_tree_ids(&program.post_inference_ir); + let post_des = collect_tree_ids(&program.post_channelize_ir); + + let relations: [(&str, &LineageMap, _, _); 2] = [ + ( + "pre-inference → post-inference", + &panes.mono_map, + &pre, + &post_inf, + ), + ( + "post-inference → post-channelize", + &panes.channelize_map, + &post_inf, + &post_des, + ), + ]; + + for (relation, map, input_ids, output_ids) in relations { + let edges = map.edges(); + assert!( + !edges.is_empty(), + "{name}: the {relation} pane relation derived no edges at all", + ); + for (u, d) in &edges { + assert!( + input_ids.contains(u), + "{name}: {u:?} is an edge origin at {relation} but not an input-pane id", + ); + assert!( + output_ids.contains(&d.id), + "{name}: {:?} is an edge target at {relation} but not an output-pane id", + d.id, + ); + } + // Dense: a node present in both panes is its own self-edge, and + // a node descends from itself — so a consumer only ever follows + // edges, never reconstructs one, and reads ancestry off the + // label it finds there. + for id in input_ids.intersection(output_ids) { + let self_edge = map.upstream(id).iter().find(|l| l.id == *id); + assert!( + self_edge.is_some_and(|l| l.labels.has_ancestry()), + "{name}: {id:?} survives {relation} without an ancestry self-edge", + ); + } + // A non-self edge is the only proof a row was consulted: a + // relation whose passes rewrote nothing derives its whole + // relation from the two pane id sets. + if edges.iter().any(|(u, d)| *u != d.id) { + exercised.push(format!("{name} / {relation}")); + } + } + } + exercised.sort(); + assert_eq!( + exercised, EXERCISED_BOUNDARIES, + "the pane relations at which the fold actually reads a row have changed", + ); + } + + /// The `program / relation` pairs at which a **blame** edge reaches the + /// pane relation — where a rewrite named blame and the fold labelled the + /// edge it contributed. See + /// [`blame_reaches_the_pane_relation_labelled`]. + const RELATING_BOUNDARIES: &[&str] = &[ + "for_accumulator / post-inference → post-channelize", + "transaction / post-inference → post-channelize", + ]; + + /// **Blame reaches the pane relation, labelled**: the `blame` column is + /// closed transitively alongside `parents`, so a consumer receives the + /// blame edges and can render or prune them. + /// + /// Pinned as the set of pane relations where such an edge exists, for the same + /// reason [`EXERCISED_BOUNDARIES`] is pinned: blame is named at four sites + /// in the compiler, two of them inside a pane relation, and a corpus or + /// recording edit that stopped exercising them would otherwise leave the + /// labelled half of the relation untested. + /// + /// A blame edge is *only* blame here: no corpus rewrite both + /// consumes a node and blames it, so nothing in the corpus pins the + /// both-labels case — the fold tests in `lineage.rs` do. + #[test] + fn blame_reaches_the_pane_relation_labelled() { + let mut relating: Vec = Vec::new(); + for (name, code) in corpus() { + let program = compile_ok(&code); + let panes = program.materialize_panes(); + let relations = [ + ("pre-inference → post-inference", &panes.mono_map), + ("post-inference → post-channelize", &panes.channelize_map), + ]; + for (relation, map) in relations { + if map.edges().iter().any(|(_, d)| d.labels.has_blame()) { + relating.push(format!("{name} / {relation}")); + } + } + } + relating.sort(); + assert_eq!( + relating, RELATING_BOUNDARIES, + "the pane relations at which blame contributes an edge have changed", + ); + } + + /// The corpus programs whose definitions inference **generalizes and then + /// specializes** — the ones monomorphization actually clones a subtree for. + /// Everything else in the corpus is first-order, and mono mints nothing. + const SPECIALIZING: &[&str] = &["generator_pipeline", "udf_chain"]; + + /// Monomorphization — the one pass inside the first pane relation — + /// explains every node it produces, on first-order and specializing + /// programs alike. + /// + /// Two recordings get it there, and both are needed: `specialize_use` sinks + /// the clone's `on_copy` pairs, and `coalesce_generalized_let` sinks the + /// chain of `let`s the binding rebuilds itself as. Without the second, a + /// specializing program leaves one unexplained `let` per demanded + /// specialization — a per-program count that tracks how many types the body + /// asked for, which is why it read as a small constant on this corpus. + /// + /// Asserted both ways so the zero cannot be vacuous: a specializing program + /// must also *kill* nodes here, since its generalized definition is + /// replaced by clones. A regression that stopped running mono at all would + /// otherwise pass. + #[test] + fn monomorphization_explains_every_node_it_produces() { + for (name, code) in corpus() { + let panes = compile_ok(&code).materialize_panes(); + let c = LeakCounts::tally(&panes.mono_leaks); + assert_eq!(c.structural(), 0, "{name}: {c:?}"); + assert_eq!( + c.unexplained, 0, + "{name}: pre-inference → post-inference is uncaptured: {c:?}" + ); + if SPECIALIZING.contains(&name) { + assert!( + c.died > 0, + "{name}: specializes, so the generalized definition must die: {c:?}" + ); + } + } + } + + /// All four passes inside the second relation explain what they produce. + /// + /// The interesting programs are the ones that drive a *whole-program* + /// rewrite, where naming one node is least obviously applicable: a + /// transaction is disassembled into a commit carrier whose pieces have no + /// single source node, and a defer cluster becomes a `LetRec` assembled from + /// contributions scattered across the body. Both are covered by recording + /// against the node each product stands in for — the `with begin():` + /// statement, the register declaration, the `let d = Defer`. + /// + /// Asserted with a non-vacuity guard for the same reason relation 1 is: a + /// program that reaches one of these passes must also kill nodes, so a + /// regression that stopped running the pass cannot pass as capture. + #[test] + fn the_second_pane_relation_explains_every_node_its_passes_produce() { + /// Reaches `transact_phase` or `channelize` — the whole-program + /// rewrites, and the last two passes to adopt the recorder. + const WHOLE_PROGRAM_REWRITES: &[&str] = &["transaction", "feed_loop", "generator_pipeline"]; + for (name, code) in corpus() { + let panes = compile_ok(&code).materialize_panes(); + let c = LeakCounts::tally(&panes.channelize_leaks); + assert_eq!(c.structural(), 0, "{name}: {c:?}"); + assert_eq!( + c.unexplained, 0, + "{name}: post-inference → post-channelize is uncaptured: {c:?}" + ); + if WHOLE_PROGRAM_REWRITES.contains(&name) { + assert!( + c.died > 0, + "{name}: rewrites its whole shape, so nodes must die: {c:?}" + ); + } + } + } + + /// Deaths are the live-set difference and nothing else: the `Died` set across + /// a pane relation is exactly `input_ids ∖ output_ids` on a real program, with + /// no pass having declared any of them. `for_accumulator` folds a mutation + /// loop into a `LetRec`, so the difference is non-empty. + #[test] + fn deaths_across_a_pane_relation_are_the_set_difference() { + let program = compile_ok(include_str!( + "../../tests/programs/for_accumulator/program.cambra" + )); + let panes = program.materialize_panes(); + let input = collect_tree_ids(&program.post_inference_ir); + let output = collect_tree_ids(&program.post_channelize_ir); + let mut expected: Vec = input.difference(&output).copied().collect(); + expected.sort_unstable(); + let mut reported: Vec = panes + .channelize_leaks + .iter() + .filter_map(|l| match l { + Leak::Died { input } => Some(*input), + _ => None, + }) + .collect(); + reported.sort_unstable(); + assert!(!expected.is_empty(), "the fixture must actually kill nodes"); + assert_eq!(reported, expected); + } + + /// A pass that rewrites the program records its rewrites under its own pass + /// tag — the tag being the one part of a row no recording site knows, and the + /// only thing that places a row in a pane relation. + /// + /// A pass that rewrites *nothing* on a given program records nothing, which + /// is the preserve case and correct (most of the corpus preserves end to + /// end), so the fixture is one that drives three of the five: the + /// transaction, which `transact_phase` disassembles, `mut_elim` rebuilds as + /// a `LetRec`, and `channelize` rewrites. + /// **Distinct predicate terms never share a `NodeId`** — with each other, or + /// with the main tree. + /// + /// Predicate ids are in the explanation domain but `assert_unique_node_ids` + /// walks the main tree only (`design/provenance.md`, "Walking the ids"), so this + /// is the only thing asserting uniqueness for them. Dedup is by `Rc` pointer + /// first: one term riding many type slots is one term and shares ids with + /// itself legitimately. + /// + /// What this catches is a rebuild that **preserves ids when it should not**. A + /// predicate cannot be mutated through its `Rc`, so every rewrite builds a new + /// `Rc` and repoints the refinement it was handed. That is a *replacement* + /// only if the walk reaches every occurrence; otherwise the original survives + /// on some type the walk missed, and preserving ids puts one id-set on two + /// live terms. `PredMemo::replacing` is the opt-in for walks that do reach + /// everything, and `uniquify` — the only one — asserts its own 1:1 + /// correspondence separately. + #[test] + fn distinct_predicate_terms_never_share_a_node_id() { + let mut found: Vec<(String, usize, &'static str)> = Vec::new(); + for (name, code) in corpus() { + let program = compile_ok(&code); + for (pane, tree) in [ + ("pre-inference", &program.pre_inference_ir), + ("post-inference", &program.post_inference_ir), + ("post-channelize", &program.post_channelize_ir), + ] { + for (_, kind) in predicate_id_collisions(tree) { + found.push((format!("{name} / {pane}"), 1, kind)); + } + } + } + assert!( + found.is_empty(), + "{} predicate id collisions: {found:?}", + found.len(), + ); + } + + #[test] + fn a_rewriting_pass_tags_its_rows_with_itself() { + let (_, code) = corpus() + .into_iter() + .find(|(name, _)| *name == "transaction") + .expect("the transaction fixture"); + let program = compile_ok(&code); + let mut recorded = program.lineage_table.recorded_passes(); + recorded.sort_by_key(|p| format!("{p:?}")); + assert_eq!( + recorded, + vec![Pass::Channelize, Pass::Letrec, Pass::Mono, Pass::Transact], + "the transaction fixture is rewritten by exactly these four passes", + ); + } + + // ----------------------------------------------------------------------- + // Perf sanity for pane capture + // ----------------------------------------------------------------------- + + /// One generated program shape, parameterized by size. + /// + /// The shapes are chosen to hit the passes the panes span: `comprehension` + /// and `arith` are `inline`-light and mostly id-preserving, `udf` drives + /// inlining and monomorphization, `loop_acc` drives `mut_elim`, and `feed` + /// drives `channelize`. + /// + /// Sizes are deliberately small. Compile time here is **superlinear in + /// program size** for reasons that predate lineage (a UDF-heavy 63-line + /// program compiles in tens of seconds), so a corpus large enough to be a + /// benchmark would be too slow to run; this is a sanity check on the + /// *ratio* between capture on and capture off, not a benchmark. + fn generated(shape: &str, n: usize) -> String { + match shape { + "arith" => { + let terms: Vec = (1..=n).map(|i| format!("{i} * {}", i + 1)).collect(); + format!("x = {}\nx\n", terms.join(" + ")) + } + "comprehension" => { + // Independent comprehensions summed, rather than a chain: a + // chained comprehension trips an unrelated substitution bug + // ("discharged binder still free after substitution into + // predicate") that has nothing to do with lineage. + let parts: Vec = (0..n) + .map(|i| format!("sum([y + {i} for y in xs if y > 0])")) + .collect(); + format!("xs = [1, 2, 3, 4, 5]\nt = {}\nt\n", parts.join(" + ")) + } + "udf" => { + let mut src = String::new(); + for i in 0..n { + src.push_str(&format!("def f{i}(x):\n x + {i}\n")); + } + src.push_str("xs = [1, 2, 3, 4, 5]\n"); + let calls: Vec = (0..n).map(|i| format!("f{i}(y)")).collect(); + src.push_str(&format!("[{} for y in xs]\n", calls.join(" + "))); + src + } + "loop_acc" => { + let mut src = String::from("xs = [1, 2, 3, 4, 5]\n"); + for i in 0..n { + src.push_str(&format!("a{i}: Mut(Int) := 0\n")); + } + src.push_str("for v in xs:\n"); + for i in 0..n { + src.push_str(&format!(" a{i} += v + {i}\n")); + } + src.push_str(&format!("a{}\n", n - 1)); + src + } + "feed" => { + let mut src = String::from("out = defer()\nxs = [1, 2, 3, 4, 5]\n"); + for i in 0..n { + src.push_str(&format!("for v in xs:\n out << v + {i}\n")); + } + src.push_str("out\n"); + src + } + other => panic!("unknown shape {other}"), + } + } + + /// The perf corpus: `(shape, size)` pairs, sized to keep the whole run in + /// the low seconds per repetition. + const PERF_CORPUS: &[(&str, usize)] = &[ + ("arith", 40), + ("arith", 80), + ("arith", 160), + ("comprehension", 6), + ("comprehension", 12), + ("comprehension", 20), + ("udf", 5), + ("udf", 8), + ("udf", 12), + ("loop_acc", 4), + ("loop_acc", 8), + ("loop_acc", 14), + ("feed", 3), + ("feed", 6), + ("feed", 12), + ]; + + /// Rough compile-time and retained-memory sanity for pane capture. Ignored + /// by default — it is a measurement, not an assertion. + /// + /// Run it as two interleaved processes so the two arms see the same machine + /// state, and take the min over repetitions: + /// + /// ```text + /// for i in 1 2 3; do + /// CAMBRA_LINEAGE=1 cargo test --release --lib lineage_pane_perf -- --ignored --nocapture + /// CAMBRA_LINEAGE=0 cargo test --release --lib lineage_pane_perf -- --ignored --nocapture + /// done + /// ``` + /// + /// With capture on it also materializes and folds both panes, since that is + /// the cost the design actually incurs. + #[test] + #[ignore = "measurement, not an assertion; see the doc comment for the driver"] + fn lineage_pane_perf() { + let capture = lineage_capture_enabled(); + let reps: usize = std::env::var(PERF_REPS_ENV) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + let mut total_compile = std::time::Duration::ZERO; + let mut total_fold = std::time::Duration::ZERO; + for (shape, n) in PERF_CORPUS { + let code = generated(shape, *n); + let mut best_compile = std::time::Duration::MAX; + let mut best_fold = std::time::Duration::MAX; + let mut rows = 0usize; + let mut rules = 0usize; + // The three retained pane snapshots are unconditional — they are not + // part of what the capture switch turns off — so their size is the + // pane design's real memory floor, against which the logs are noise. + let mut panes_nodes = 0usize; + for _ in 0..reps { + let t0 = std::time::Instant::now(); + let program = compile_ok(&code); + best_compile = best_compile.min(t0.elapsed()); + rows = program.lineage_table.len(); + rules = program.lineage_table.rule_count(); + panes_nodes = collect_tree_ids(&program.pre_inference_ir).len() + + collect_tree_ids(&program.post_inference_ir).len() + + collect_tree_ids(&program.post_channelize_ir).len(); + if capture { + let t1 = std::time::Instant::now(); + let panes = program.materialize_panes(); + std::hint::black_box(&panes.post_channelize); + best_fold = best_fold.min(t1.elapsed()); + } + } + if !capture { + best_fold = std::time::Duration::ZERO; + } + total_compile += best_compile; + total_fold += best_fold; + eprintln!( + "[perf capture={capture}] {shape}/{n}: compile {:?} fold {:?} rows {rows} \ + rules {rules} pane_nodes {panes_nodes} lines {}", + best_compile, + best_fold, + code.lines().count(), + ); + } + eprintln!("[perf capture={capture}] TOTAL compile {total_compile:?} fold {total_fold:?}"); + } + /// Driver that runs `compile_program` for an error-only test, returning /// the collected error list. Discards the program — these tests only /// care about which errors surface. @@ -1274,7 +2444,7 @@ Error: lowering error let pointed = &code[span.start..span.end]; assert_eq!( pointed, "1 + \"a\"", - "the blame is the node whose coalesce frame raised the error — here the \ + "the blame is the node whose coalesce rule raised the error — here the \ `+` application over both operands, not the whole program or nothing" ); } diff --git a/src/ccl/design/README.md b/src/ccl/design/README.md index 2dc86c45..19cd09e0 100644 --- a/src/ccl/design/README.md +++ b/src/ccl/design/README.md @@ -29,7 +29,7 @@ CHL source | [type-inference.md](type-inference.md) | Cambra's inference algorithm: the two-pass emit → coalesce engine (`ccl/infer/`), the constraint solver (`ccl/infer/solver/`), let-polymorphism, dependent Pi types and refinements, and post-inference validation. | | [lowering.md](lowering.md) | CHL → CCL lowering: how comprehensions, lambdas, `def`s, and generators become CCL shapes, and the surface syntax of the deferred-collection operators. | | [optimization.md](optimization.md) | The optimization/compilation passes: inlining, lambda elimination, join/aggregate planning, algebraic simplification, and conversion to tile operators. | -| [provenance.md](provenance.md) | How a node keeps its link to the source the user wrote across the whole pipeline: the `NodeId`/`Pass` identity primitives, the `RewriteStep` lineage model and its collapse, the recorder, the always-on lowering projection release diagnostics read, and what the inspector consumes. | +| [provenance.md](provenance.md) | How a node keeps its link to the source the user wrote across the whole pipeline: the `NodeId`/`Pass` identity primitives, the `LineageTable` lineage model and its collapse, the recorder, the always-on lowering projection release diagnostics read, and what the inspector consumes. | Provenance is the one cross-cutting concern in the table: every pass above both preserves node identity and records what it rewrote, so diff --git a/src/ccl/design/lowering.md b/src/ccl/design/lowering.md index d557d05f..fe8965f2 100644 --- a/src/ccl/design/lowering.md +++ b/src/ccl/design/lowering.md @@ -132,7 +132,7 @@ Feed channelization runs **after** `infer` and `mut_elim::run` (and after `inlin In broad strokes, for each cluster of consecutive `let d_i = Defer in …` bindings the step walks the cluster body to extract every `Feed(d_i, V)` and the (at most one) `Define(d_i, V)`, combines the extracted values via `++` (`TypedExprNode::Copair` — the channels have distinct index sets), and emits the cluster as one **mutually-scoped `Feed`-kind `LetRec` group** — so cross-defer references (`x ≪= y; y ≪= …`) need no binding order, and a reference *cycle* among channels is rejected by the letrec guardedness rule (channels carry no guard). Recognition later flattens the acyclic group to dependency-ordered `let`s. Special handling exists for per-iteration feeds (Compose/Apply with iteration lambda), N-arm filter-feed Cases (`if`/`elif` fan-out to refined-source channels), and a handful of structural rewrites (defer-returning let lift, alias inlining for defer handles). Because `mut_elim` hoists every in-loop feed to a top-level `Feed(defer, view)` before this step runs, channelization is origin-agnostic — it never distinguishes an accumulator-loop feed from a feed-only-loop or scalar feed. -The design of record — where this step sits in mutability elimination and why `desugar_defers`/`retype` were retired — is **[mutability.md](mutability.md) §4**; the in-depth implementation notes (extraction paths, shadow-renaming, error modes, navigation map) live in the `ccl/channelize.rs` module rustdoc. +The design of record — where this step sits in mutability elimination and why `channelize_defers`/`retype` were retired — is **[mutability.md](mutability.md) §4**; the in-depth implementation notes (extraction paths, shadow-renaming, error modes, navigation map) live in the `ccl/channelize.rs` module rustdoc. ### Inference before desugaring diff --git a/src/ccl/design/provenance.md b/src/ccl/design/provenance.md index 6fce36ba..d63360aa 100644 --- a/src/ccl/design/provenance.md +++ b/src/ccl/design/provenance.md @@ -6,17 +6,92 @@ at lowering, monomorphization cloning subtrees, inline fanning UDF bodies out, channelize rewriting defers, lambda-elim synthesizing combinators, planning fusing clauses). -A **pane** is a snapshot of the AST at one point in compilation. `CompiledProgram` retains three — -`pre_inference_ir`, `post_inference_ir`, `post_desugar_ir` — and the inspector renders each in one -UI pane, which is where the name comes from. **Below** a pane means later in the pipeline, on a more -lowered tree; it is not tree depth and not a layering. - **Status markers.** The substrate — the identity primitives, the lineage model, -the recorder, and the always-on lowering projection — is in tree. Everything a -**Planned** marker introduces is designed but not yet built: the passes' adoption -of the recorder, the pane-boundary folds, and the inspector's consumption of -them. A reader on `main` can tell the two apart by the marker alone; unmarked -prose describes code you can go read. +the recorder, the passes' adoption of it, the always-on lowering projection, the +`NodeId`-keyed table that is the sink +([The node table](#the-node-table-srcccllineagers)), and the fold over it — is in +tree. Everything a **Planned** marker introduces is designed but not yet built: +the inspector's consumption of the panes. A reader can tell the two apart by the +marker alone; unmarked prose describes code you can go read. + +Adoption is complete across the five passes the two pane relations span — `Mono`, +`Inline`, `Transact`, `Letrec`, `Channelize` — and the gate holds at zero on both. +Three further sites record without reaching any table in a normal build, because +`compile_program` opens no `PassScope` around them: `simplify`, +`planning/iterate`, and `transact_phase`'s as-of-read rewrite. `lambda_elim` +records nothing at all, and operator conversion has no identity to record +against; see +[Known prerequisites for panes past `post-channelize`](#known-prerequisites-for-panes-past-post-channelize). + +## Mechanism at a glance + +Every node carries a `NodeId`, unique for the process and preserved across a +rewrite that keeps the node. Identity is the only thing the pipeline threads; the +rest is derived from it. + +A rewrite **records**: it names the node it is about to rewrite, and every node +minted while that recording is the innermost open one takes the named node as a +parent. A site declares nothing else. It does not declare what it produced — the +produced side is discovered through the mint and copy hooks — and it does not +declare what it destroyed, because a death is the difference between the ids live +before and after. + +Those rows accumulate in one `NodeId`-keyed table per compile, one row per node: +its `parents`, its `blame`, and an interned rule tag. Folding the rows of a chosen +set of passes turns the table into a relation between the trees at each end of +those passes, labelling every edge and reporting the ids it could not explain. + +Two things fold. A **pane** is a retained snapshot the inspector displays, +materialized after a set of passes; the fold over the passes between two adjacent +panes is the **pane relation**, and the leak classes are asserted empty there on +every compile. A `LineageAudit` folds between two chosen points against the live +tree rather than a snapshot, and measures instead of gating: it is how a pass's +recording is checked before any pane spans it, and how a suspected gap is located +without waiting for the gate to be extended. + +Source spans enter at one place. Lowering projects every id `collect_tree_ids` +enumerates onto the span of the CHL it came from, and every later span is derived +by walking `parents` back into that projection. + +## Terms + +| term | what it is | +|---|---| +| **pane** | A retained AST snapshot the inspector displays, materialized after a set of passes. Each one costs a retained full-tree clone. | +| **pane relation** | What folding the passes between two adjacent panes produces: an id-to-id relation with labelled edges. The **durable, gated** artifact — the leak classes are asserted here. | +| **recording** | The scope `lineage::enter` opens over one rewrite, held as a `FrameGuard`. Every node minted while it is the innermost open one takes the node it names as a parent. Prose here says "a recording" for the scope, "the recording site" for the code location, and "records against X"; the guard is the RAII value that closes it. | +| **slot** | The node a recording names — `lineage::enter(slot_id, …)` — read off the tree *before* the rewrite runs. Normally a main-tree node. A predicate interior *may* be one, but work **on** a predicate is usually recorded against the predicate's own root, and work that *produces* one against the main-tree node whose type will carry it. | +| **predicate interior** | A `NodeId` on a `TypedExpr` inside a `Type::Refinement`'s predicate. Ordinary ids from the same counter, and inside the id domain a fold must explain: `collect_tree_ids` enumerates them. They are the one place explanation and uniqueness come apart — `assert_unique_node_ids` walks the main tree only, because a predicate interior may legitimately carry a main-tree id. See "Walking the ids". | + +An audit's endpoint is **chosen**, because a span running past the last +instrumented pass counts everything the uninstrumented tail mints as a defect — a +number that cannot reach zero however correct the recording is, which makes the +audit read as a broken gate rather than a measurement. The `full` span therefore +ends at the last instrumented pane, `post-inference..post-as-of-read`, stopping in +front of `lambda_elim`. + +**The same discipline governs the gate.** `CAMBRA_LINEAGE_GATE=1` makes *every* +compile fold its pane relations and gate the leak classes, so the gate's corpus +becomes whatever the caller compiles — point it at the test suite and it covers +every program there instead of the handful `context.rs`'s `corpus()` lists. That +sample is what let two recording gaps live: `transact_phase` calling +`mut_elim::fold_induction_loop` with nothing recording, and `flatten_spine`'s +value-position writer hoist. Both are shapes the eleven listed programs do not +have. CI runs with it on; it costs about 4% of the test step. + +It gates `gated_pane_relations()`, not both, for the reason above: the first +relation spans monomorphization and inference, and **inference's predicate +producers do not record**. `specialize_use` clones a definition per +instantiation, and the copies of a predicate term inside it row against interior +ids no pass produced. Over `tests/compilation_pipeline` that is 5 programs, all +UDF-with-filter or poly-wrapper shapes. Gating it today would report a constant, +not a regression. **Add it to `gated_pane_relations()` in the commit that makes +inference record**, the same way an endpoint moves. + +**Move the endpoint when a pass becomes instrumented, in the commit that +instruments it:** to `post-lambda-elim` when the elim pass records, and to +`join-planned` when planning does. Leaving it behind understates coverage; moving +it ahead reintroduces the unreachable zero. > This file is the reference for the shipped shape and wins on what the code > does. The decision log behind it — the rejected alternatives, the measurements, @@ -25,199 +100,432 @@ prose describes code you can go read. ## Node identity (`src/ccl/provenance.rs`) -**`NodeId`** is a `Copy` newtype giving each IR expression node an identity: stable, never reused, -and off its own atomic counter. It rides inline on `TypedExpr`. Lineage and source attribution are -both keyed by it, which is what lets an inference error on a node minted three passes ago resolve to -the source span the user wrote. It is distinct from `Uid` because `Uid` identifies binders and a -`NodeId` identifies expression nodes. - -A node is **live** when it is reachable from the expression tree a pass hands on — the node set -`walk_children` enumerates, called the **main tree** throughout this doc. A refinement predicate is -a term hanging off a *type* slot, so its nodes are reachable from the tree without being in it; the -distinction is what the two walks below split on. - -Four properties define a `NodeId`. - -- **It is not part of a node's value.** `TypedExpr`'s hand-written `PartialEq` skips `node_id`, so - two structurally-equal nodes compare equal with distinct ids, which the passes' - structural-equality checks depend on. Nodes are never hashed by value: `TypedExpr` has no `Hash` - impl. `NodeId` itself is `Hash`/`Ord`, as a map key. -- **Two live nodes never share one.** This is what makes an id an identity rather than a label. Two - nodes at one id collapse to a single entry in every `NodeId`-keyed walk, and give the - `SourceProjection` — the `NodeId → SourceAttribution` map the lineage fold produces - ([The collapse](#the-collapse)) — one attribution for two nodes. -- **Construction and copying both mint.** `Expr::new` mints, and `Clone` mints for every node it - copies. A call site duplicating a subtree gets distinct identities without asking for them, so no - site has to work out which of its copies is the survivor — reaching a shared id takes writing one - through a named primitive ([Duplication](#duplication)). -- **Uniqueness is asserted on the main tree only.** `assert_unique_node_ids` walks children and - stops there, so predicate interiors are outside the uniqueness walk. - -`NodeId::PLACEHOLDER` is the reserved sentinel for `Default`/`mem::take` throwaways. The recorder — -the ambient session that logs every mint and copy ([The recorder](#the-recorder)) — ignores it, and -`assert_unique_node_ids` backstops that it never persists into a checked tree. - -### Maintaining uniqueness - -`Clone` freshens: it mints a new `NodeId` for every node it copies and reports each `(origin, -fresh)` pair through `on_copy`, the recorder's copy hook. The alternative is a call site that -decides — keep the id here, freshen there — and a site that decides wrong puts two nodes on one id, -which surfaces at a boundary assert far from the site, if at all. Freshening removes the decision -rather than answering it. - -`assert_unique_node_ids` enforces uniqueness at every pass boundary in `compile_program`: -post-lowering, -inline, -transact, -letrec-run, -desugar, -as-of-read, -lambda-elim, -planning, -gated on `cfg!(any(debug_assertions, test))`. The walk is `O(nodes)` per boundary and compiles out -of a release build, along with the leak checks (`assert_leaks_clean` is gated the same way, so -nothing enforces either property in release). The lineage fold that produces the release-critical -projection — [The collapse](#the-collapse) — stays always-on. A boundary check states a property of -the tree rather than of the pass that produced it, so reordering the passes leaves the checks where -they are and still bounds every pass between two of them. A clean run is therefore evidence about -the boundaries, not about any individual pass. +- **`NodeId`** — a `Copy` newtype giving each IR expression node a stable, + never-reused identity (its own atomic counter, distinct from `Uid`). It rides + inline on `TypedExpr`, whose hand-written `PartialEq` **skips it**: provenance + is metadata, not part of a node's value, so two structurally-equal nodes stay + equal even with distinct ids — which the passes' structural-equality checks + depend on. (Nodes are never hashed by value: `TypedExpr` has no `Hash` impl. + `NodeId` itself is `Hash`/`Ord`, as a map key.) `NodeId::PLACEHOLDER` is the reserved sentinel for + `Default`/`mem::take` throwaways (ignored by the recorder; `assert_unique_node_ids` + backstops that it never persists into a checked tree). +- **`Pass`** — the compiler stage that produced/rewrote a node (`Lower`, + `Uniquify`, `Inline`, `Channelize`, `Transact`, `Letrec`, `Mono`, `LambdaElim`, + `Planning`). It lives in the lineage *data* (each row's `via`), never in a + type. ### Walking the ids -Three walks answer three questions. - -| Walk | Question | Domain | -|---|---|---| -| `assert_unique_node_ids` | may two live nodes share an id? | children | -| `collect_tree_ids` | which ids must the lineage fold account for? | children, plus refinement predicates | -| `uniquify::collect_node_ids` | did a `PredMemo` rebuild drop or re-mint an id? | the same, as a multiset | - -`collect_tree_ids` reaches a predicate through a type slot, a `user_annotation`, or a `Cast` target. -It is the operative definition of what the fold must explain: the leak classes and every -`SourceProjection` enumerate from it, so a node it returns is a node the fold explains or reports as -a leak. A refinement predicate is program text the user wrote — `[x for x in xs if x > k]` puts `x > -k` in one — so it earns the same attribution as any other node. Lowering sweeps a finished predicate -through `LoweringContext::tag_predicate`, which is what makes a guard error resolve to a caret. - -That sweep is the predicate domain's **entry** crossing, and the only crossing that is recorded. A -predicate being rewritten (uniquify and inference rebuild them through a `PredMemo`) and a predicate -being raised back into the main tree (planning) both still mint under no recording. No boundary -check reads the ids they mint: the uniqueness walk stops at children, and the leak checks run only -at the lowering boundary, upstream of both crossings. - -`uniquify::collect_node_ids` checks multiset preservation across uniquify's own `PredMemo` rebuilds, -which is where ids could be dropped or re-minted. - -`Rc` sharing is load-bearing and constrains how recording may be done. One predicate term rides many -type slots as a shared `Rc`, and planning's compile memo is `Rc`-keyed, so splitting the sharing -compiles one predicate once per occurrence. Recording is therefore idempotent per id rather than per -slot the id is reached through: `lowering_predicate_leaf` skips an id already recorded, which also -stops a sweep replacing precise attribution with a coarse label. For the same reason a duplication -path may share a predicate `Rc` with its source rather than rebuild one (see -`design/type-inference.md`, "Sharing is an invariant, not an optimization detail"). +**Two questions, two domains.** Keep them apart, because they used to have the +same answer and no longer do: + +- **Explanation** — *which ids must the fold account for?* The main tree **and + its refinement predicates**. `collect_tree_ids` enumerates both (children, + plus the predicate reachable through a type slot, a `user_annotation`, or a + `Cast` target) and is the operative definition: the fold's leak classes and + every `SourceProjection` enumerate from it, so a node it returns is a node the + fold must explain or report as a leak. +- **Uniqueness** — *may two live nodes share an id?* The main tree, and nothing + else. `assert_unique_node_ids` walks children only, and deliberately: a + predicate interior may legitimately alias a main-tree id at inline's blind + spot, so a predicate-inclusive uniqueness walk would false-fire. Uniqueness + *across distinct predicate terms* is asserted instead, by the corpus test + `distinct_predicate_terms_never_share_a_node_id`. + +Predicates were previously outside *both* domains — "carried, never checked" — +and free to alias their source's ids. Being in the explanation domain is a +deliberate change: a refinement predicate is program text the user wrote +(`[x for x in xs if x > k]` puts `x > k` in one), so it deserves the same +attribution as any other node. The "cost, accepted" this section used to record — +a guard error reporting without a caret, because its id was not in the lowering +projection — is exactly what explanation buys back. + +Three crossings, and each one has to record: + +1. **Entry** — a term is put *into* a predicate. Lowering's three + `refined_data_fun` sites sweep the finished term through + `LoweringContext::tag_predicate`; inference's `singleton_predicate` is + recorded against the literal's own node, which is also the edge that links + a singleton refinement back to the literal the user wrote. +2. **Transformation** — a predicate is rewritten. The rewritten term *replaces* + the original in its `Refinement`, so it is the same logical node. +3. **Raising** — a predicate is materialized back into the main tree, which + planning does. **Not yet recorded**, and not urgent here: those nodes are + minted below the last pane, so nothing gates them. Lands with the planning + commit upstack. + +`Rc` sharing stays load-bearing and constrains how recording may be done. One +predicate term rides many type slots as a shared `Rc`, and planning's compile +memo is `Rc`-keyed, so splitting the sharing compiles one predicate once per +occurrence. Recording must therefore be idempotent **per id**, not per slot the +id is reached through: `lowering_predicate_leaf` skips an id already recorded, +which also stops a sweep replacing precise attribution with a coarse label. + +`uniquify::collect_node_ids` is a third walk for a third question: a debug +tripwire asserting **multiset preservation** across uniquify's own `PredMemo` +rebuilds. It is neither explanation nor uniqueness, and it deliberately does not +dedup by `PredicateId`. + +`distinct_predicate_terms_never_share_a_node_id` asserts the uniqueness property +predicates can satisfy: dedup by `Rc` pointer first, then require the ids of the +deduped set to be distinct. One term riding N slots is one term and shares its +ids with itself legitimately, while two *different* predicate terms sharing an id +is a defect. What it catches is a rebuild that preserves ids when the walk did not +reach every occurrence. ### Duplication -Three primitives, chosen by what the copy denotes. - -- **`clone`** — the copy is a *sibling*: same value, distinct identity, `annot(p) = annot(o)`. The - default. Every re-minted node fires `on_copy`, so a freshen is captured as `Op::Copy`, the - lineage step whose outputs mirror an origin's history - ([The lineage model](#the-lineage-model-srcccllineagers)). Capture is live the moment a session is - installed and a no-op before that; no call site needs to know which. -- **`clone_preserving_ids`** — the copy *is the same node*, so it keeps its ids. Sound because the - copy is never reachable from a tree beside its source, and narrow: a `Subst` discharge template, - which is not a tree node and is copied again at every read; a throwaway the normal path discards, - such as a rollback copy or a scratch tree; and a test comparing trees across a pass. Not a way to - silence a **leak**, the fold's report of a node whose history it cannot account for - ([The collapse](#the-collapse)): an `Unexplained` or a `CopyOfUnknown` means a copy was made with - no step open, or against an origin the log never recorded, which is a recording gap whose fix is - to record the copy. -- **a copy at the occurrence's id** — the root takes an id the tree already holds and the interior - freshens: substitution, and the one copy that shares an id. The replacement for a `Var(𝑥)` - occurrence denotes what the occurrence denoted, the value of 𝑥 at that position, so attribution - there is the occurrence's; N reads give N subtrees under N ids the tree already holds. One site, - `Subst`'s compound-replacement arm in `as_expr_preserving`, which carries the alternative it is - chosen over: recording the copy against the occurrence instead of sharing its id resolves to the - same entry, at a death per occurrence and an ordering constraint on the record. - -**Freshen at placement, not at construction.** Most passes build intermediate structures — guard -vectors, path conjunctions, per-branch environments — whose entries are aliased and then copied into -the output. Placement is where a copy's multiplicity is known; freshening earlier assumes an answer, -and the three outcomes carry different costs. - -- **Moved** into the output: no cost. The output node holds the fresh id, and its parent is the - original. -- **Copied again** into the output: a defect the *lineage* fold catches, not the uniqueness walk. - The intermediate copy is stranded, so the placed copy's `Op::Copy` names an origin no node holds, - walking `parents` for a span dead-ends, and the fold reports `Leak::CopyOfUnknown`. Freshening the - substitution engine's `Subst`-resident templates produces exactly this. -- **Dropped**: one spent id and one row no query reaches. No check reports it — both leak checks - enumerate from the tree, there is no produced-side check (below), and a node absent from the tree - is outside `assert_unique_node_ids`. Construction is the only constraint: `new` mints and records, - `preserve` carries. - -**A term crossing out of the predicate domain must not land aliased ids.** The uniqueness walk does -not reach predicate interiors, so a pass lifting one into the main tree owes a freshen at the point -of entry — `planning::iterate`'s `fn_of_bare_predicate` lift does exactly that. A lift that -*rebuilds* the term is already safe: `planning::groupby`'s key extraction goes through -`lambda_elim::run`, which re-mints every node. The requirement is that nothing aliased arrives; -rebuilding is one mechanism that satisfies it. `groupby_recognition_lifts_the_key_without_aliasing` -pins the property at the group-by site, so an elim that started preserving ids fails there rather -than at a boundary. The `iterate` lift has no such test. +**Every pass yields unique `NodeId`s.** `Clone` is hand-written and +**freshens**: it mints a new `NodeId` for every node it copies and reports each +`(origin, fresh)` pair through `on_copy`. Two live nodes therefore cannot share an +identity by accident, and sharing one is something a site has to ask for. + +That inverts the standing hazard rather than removing the reason for the rule. A +derived `Clone` copied `node_id`, so a bare `.clone()` of a subtree landing at two +live positions emitted two nodes with one identity — collapsing them to one entry +in every `NodeId`-keyed walk, giving the `SourceProjection` one attribution for +two nodes, and making a `NodeId → OperatorId` map non-functional. Uniqueness is +still what makes an id an *identity* rather than a label; it is just no longer +the call site's job to remember. + +`assert_unique_node_ids` enforces it at every pass boundary in `compile_program` +— post-lowering, -inline, -transact, -letrec-run, -channelize, -as-of-read, +-lambda-elim, -planning — gated on `cfg!(any(debug_assertions, test))`. The walk +is `O(nodes)` per pass boundary and buys nothing in a release compile, where the +fold's leak classes cover the same ground. The check is a tree invariant and +encodes no pass order, so a reordered pass carries its check with it. It also +means a pass is implicated only at a boundary that looks at it: a clean run is +evidence about the gates, not about the passes between them. + +Three shapes, and the choice between them is about what the copy *denotes*: + +- **`clone`** — duplication, and the default. The copy is a *sibling*: same + value, distinct identity, `annot(p) = annot(o)`. Every re-minted node fires + `on_copy`, so an open recording rows the copy on the node it duplicated, and + with none open nothing is written. No call site needs to know which. +- **`clone_preserving_ids`** — the copy *is the same node*, so it keeps its ids. + Sound because the copy replaces or shadows its source: the two are never both + reachable from one tree. Two shapes qualify — a snapshot taken for rollback or + comparison, which the normal path discards, and a test comparing trees across a + pass — plus the moves-out-of-a-borrow, where Rust forces a copy and the source + is dropped. + + **Not a way to silence a leak.** An `Unexplained` or `ParentUnknown` means a + copy was made with nothing recording, or against an origin the table never + recorded. That is a *recording* gap, and the fix is to record around the copy. +- **`clone_at`** — root-carry, for substitution. The replacement for a `Var(𝑥)` + occurrence denotes what the occurrence denoted — the value of 𝑥 *at that + position* — so the occurrence keeps its own id while the interior becomes a + fresh node-set. N reads give N distinct roots, so uniqueness holds without + deleting the read sites from the output. `Subst`'s compound-replacement arm is + the engine. + + The root is built **at** the carried id rather than minted and overwritten, so + carrying costs nothing. The earlier `clone().re_root(id)` spelling minted a root + id, fired `on_copy` for it and then discarded it — one stranded id and one + stranded row per substituted occurrence. `re_root` is deleted; a field-wise + rebuild that wants an existing id uses `preserve`, which mints nothing either. + +**Freshen every copy, not all-but-one-by-position.** Which copy retains the +original id is a *fate* question, and keep-first guesses it wrong exactly when +position 0 is the copy that later dies. Freshening needs no knowledge of +downstream fates: the original either survives in place and keeps its id, or dies +and is consumed by the rewrite that dropped it. (Lowering's `fan_out_copy` is +keep-first on purpose — it is the pass that *mints* the originals, so position 0 +is the source image by construction.) + +**Freshen at placement, not at construction.** Most passes build intermediate +structures — guard vectors, path conjunctions, per-branch environments — whose +entries are aliased and then copied into the output. Placement is where a copy's +multiplicity is known; freshening earlier assumes an answer, and the three +outcomes carry different costs. + +- **Moved** into the output: no cost. The output node holds the fresh id, and its + parent is the original. +- **Copied again** into the output: a defect the gate catches. The intermediate + generation is stranded, so the placed copy names an origin no node holds, a + `parents` walk for a span dead-ends, and the fold reports `ParentUnknown`. + Freshening the substitution engine's `Subst`-resident templates produces + exactly this, and the pane-relation gate fails. +- **Dropped**: one spent id and one row no query reaches. No check reports it — + both leak checks enumerate from the tree, there is no produced-side check + (below), and a node absent from the tree is outside `assert_unique_node_ids`. + Construction is the only constraint: `new` mints and records, `preserve` + carries. + +**A term crossing out of the predicate domain must not land aliased ids.** +Predicate interiors are outside the *uniqueness* domain (above) and may already +alias main-tree ids, so a pass that lifts one into the term tree owes a freshen at +the point of entry — `planning::iterate`'s `fn_of_bare_predicate` lift does +exactly that. A lift that *rebuilds* the term is already safe: +`planning::groupby`'s key extraction goes through `lambda_elim::run`, which +re-mints every node. That is the mechanism, not the requirement; the requirement +is that nothing aliased arrives, and each site pins it with a test. ## The lineage model (`src/ccl/lineage.rs`) Recording is a **byproduct of performing a rewrite**, never a post-pass diff. As -a pass runs it appends `RewriteStep`s to a `LineageLog`: - -- `Op::Transform { consumed, produced }` — inputs vanish, outputs appear (empty - `produced` = discard; an id in `consumed ∩ produced` survives while absorbing). -- `Op::Copy { origin, produced }` — outputs mirror `origin`'s lineage (freshened - duplicates); silent on the origin's own fate. - -**`Pass`** names the compiler stage that produced or rewrote a node (`Lower`, `Uniquify`, `Inline`, -`Desugar`, `Transact`, `Letrec`, `Mono`, `LambdaElim`, `Planning`). It lives in the lineage data, as -each step's `via`, and never in a type. - -Each step *separately* carries a `blame` set (the upstream ids the outputs -attribute to — **not** the same as `consumed`), a `nature` (the trinary fidelity -axis `Source` / `Expansion` / `Machinery`), and a stable `label`. `consumed` -drives fate/leak accounting; `blame` drives span resolution; the two are never -mixed. `Nature::Source` — the root of a lowered source expression — is emitted -**only by lowering**; the fold's attributing arms carry a debug guard that no -pass step ever carries it. - -Lowering records the same way, into its own `LoweringLog` of `LoweringStep`s. -A `LoweringStep` is structurally a `RewriteStep` whose attribution channel is a -literal source-span `anchor` (attached at construction) rather than a NodeId -`blame` (resolved later through the accumulating projection) — that -attached-literal-vs-resolved-reference difference is why the two are sibling -structs, not one generic. Its log is folded **once** at the lowering boundary by +a pass runs, every node it *produces* gets a row in the `LineageTable`: + +| column | what it holds | +|---|---| +| `parents` | the ids the rewrite consumed to produce this node: the node the recording named, or a fusion's whole consumed set | +| `blame` | the upstream ids the node is *related to but did not consume* — **not** the same as `parents` | +| `rule` | an interned `RewriteTag` — the `{via, nature, label}` triple | + +**No column records a fate.** Both columns make a claim about the *product* — one +that it was consumed from these ids, one that it is about them — and neither says +whether any named id survived. Deaths come from one place only: `input_ids ∖ +output_ids` across a pane relation. Nothing predicts a fate, so no pass can +over-claim one. + +That silence is what makes *adopt-a-live-subtree* expressible: a rewrite that +keeps a node's id while minting a wrapper over one of its children must not be +read as declaring that node dead. It is equally what makes a fusion's parent set +safe — naming a survivor there costs an over-broad edge, never a phantom death. + +The parents column's **cardinality** carries the rewrite's shape, and nothing +else about the shape is recorded: one parent is a 1:1 or 1:many rewrite (several +products each rowing on the same slot), several parents is a genuine fusion. A +node cannot be its own parent — an in-place rewrite that keeps its id is a +*preserve*, which records nothing, because identity here is referent identity +and the pane resolves it by shared id. The *closure* of `parents` is reflexive +even so, which is why a surviving node carries an ancestry self-edge +([The edge labels](#the-edge-labels)). + +Attribution resolves through **`parents` ∪ `blame`** — parentage first, then +blame's distinct additions, so the span order is deterministic and blame is never +dropped when it names a node the parents do not. Blame is named at four sites in the whole tree, so for almost +every node this is simply the spans of what it was made from, which is why +walking the lineage recovers a source location at all. + +**Two labels on one relation, which is why they stay separate columns.** Both +relate a node to other nodes; what differs is what the relation *asserts* — a +label on the edge, not the presence of one. `parents` says **descends from**; +`blame` says **related to, but not consumed**. Blame may name a node that +*survives* the rewrite, so welding it into `parents` would answer "what was this +made from" with a live node elsewhere in the tree. Both columns reach the +pane relation, each labelling the edges it contributes, and the leak audit reads +the ancestry label alone (see +[The edge labels](#the-edge-labels)). + +Keeping the two apart is what lets a consumer render each faithfully — show the +blame, or prune it — rather than receiving one unlabelled edge set it +cannot take apart again. `mut_elim`'s `enter(stmt_id)` + `blame(for_id)` is the +shape: the products descend from the statement and are *about* the loop keyword. + +`Nature::Source` — the root of a lowered source expression — is emitted **only by +lowering**; the fold's attributing helper carries a debug guard that no pass row +ever carries it. + +Three invariants are properties of the *write*, and are asserted at +`LineageTable::record`, at the site that would violate one, rather than in a +fold that only runs when a pane is materialized: **one row per id** (attribution +has no join, so a second claimant has no answer), **every row anchored through +some channel** (consumption or blame — a row with neither cannot explain where +its node came from), and **no node is its own parent**. All three are +`debug_assert!`s: `record` is the construction hot path, so what is gated is the +checking, never the row. + +Lowering records separately, into a `LoweringLog` of `LoweringStep`s, because +its attribution channel is different in kind: a **literal source span attached +at construction** (lowering knows the source token it is imaging right there) +rather than a NodeId reference resolved later through the accumulating +projection. A `LoweringStep` is therefore not a row but one of the two shapes +lowering actually has — a **leaf mint** (one id, one span, a nature and a label) +or a **copy** (an origin and its freshened duplicates, mirroring the origin's +folded entry verbatim). Its log is folded **once** at the lowering handoff by `collapse_lowering` into the always-on lowering projection (below). ### The recorder -An ambient thread-local step stack (`STEP_STACK`) + an installed log -(`ACTIVE_LOG`), mirroring `infer_var::ACTIVE_ARENA`: - -- `Expr::new` calls `on_mint`; the freshen helpers call `on_copy` — so a `step(…)` - RAII guard open around a rewrite *captures* the births/copies in its dynamic - extent (innermost frame wins). Empty stack ⇒ recording off (a cheap emptiness - check on the construction hot path). -- `RecorderSession` installs/drains the log at a pass boundary. - -**Planned — the passes' adoption.** Production step sites in `infer` (mono — the -`specialize_use` clone `Copy`s and the `coalesce_generalized_let` wrapper -`Transform`), `inline` (beta/alias discard `Transform`s + fan-out `Copy`s), and -`channelize` (cluster/feed-union/lift/drop). Each boundary in `compile_program` -wraps the pass in a `RecorderSession` and retains its drained `LineageLog` on -`CompiledProgram::pass_lineage`, in pipeline order: `[(Mono, …), (Inline, …), -(Transact, …), (Letrec, …), (Desugar, …)]`. - -Today the only production step frames are lowering's two `Copy`-capture frames -(below); every other rewrite runs unrecorded. +An ambient thread-local stack of open recordings (`STEP_STACK`) plus an installed +sink, mirroring `infer_var::ACTIVE_ARENA`: + +- `Expr::new` calls `on_mint` and the freshen helpers call `on_copy`, so a + recording open around a rewrite *captures* the births and copies in its dynamic + extent, innermost first. An empty stack means recording is off, which costs one + emptiness check on the construction hot path. +- `TableSession` installs the table for a **whole compile**; `PassScope` names + the pass rows are tagged with, for **one pass**. The two nest that way because a + row's key is a process-unique `NodeId` and needs no pass set to disambiguate + it, while a `RewriteTag` needs a pass the recording site cannot supply: the site + knows its `label` and `nature` but not which pass is running, and the pass + boundary that opens the scope knows exactly that. +- `LoweringSession` installs lowering's log instead, and is always-on. + +**A site declares nothing.** It names the node it is *about to rewrite*: + +```rust +let _g = lineage::enter(slot_id, "inline.beta", Nature::Machinery); +``` + +`slot_id` is read off the node before the rewrite runs; every id minted while the +guard is innermost gets a row naming `slot_id` as its parent. The pairing is +**(id before, minted during)**, not (value in, value out), so one recording fits +an `fn(Expr) -> Expr` rewrite and an `&mut Expr` one alike. A recording that mints +nothing writes nothing — that is the preserve case. + +Two escape hatches are **inherent methods on the guard**, so a site can only ever +address the recording it holds (each debug-asserts it is the innermost open one, +turning a channel fired into a callee's recording or an enclosing recursion's into +a loud failure rather than a silent misattribution): + +- `also_consumes(id)` — genuine fusion (many:1), the only thing that puts a + second parent on a row and the only place any id is named at record time. +- `blame(ids)` — nodes this rewrite is **related to but did not consume to + produce** its outputs. Attribution unions them with the parents; the lineage + relation carries them as *blame* edges. Blame is named at four sites in + the whole tree, so `parents` alone is what recovers a source location for + almost every node. + +`enter` is the only constructor a pass uses. `copy_frame` is the one recording +that names **no** node: uncurry's template-interior freshens and the compare-chain +operand freshens duplicate nodes with no slot being rewritten, and each captured +copy carries its own origin from the hook. A recording that names no node has +nowhere to attach a mint or a consume, which `OpenStep::assert_copy_only` +enforces. + +**Where recordings are open today.** Lowering's leaf appends and copy sinks, plus +recordings inside the two pane relations: `infer/solve` (`mono.specialize`, +`mono.coalesce_let`), `infer/emit` (`infer.lit_singleton`), `inline` +(`inline.alias`, `inline.udf`, `inline.beta`), `mut_elim` (`letrec.loop`, +`letrec.bare_write`, `letrec.hoist_writer_body`, `letrec.terminalize_write`), +`transact_phase` (strip, unwrap block, writer, commit record, history binding, +key rebind, key-init stash, carrier, the cross-domain and await-final rules), and +`channelize` (`channelize.cluster`, `channelize.defer_lift`, +`channelize.defer_collapse`). Two shared helpers record under whichever pass +scope is open around them: `subst` (`subst.vacuous`, `subst.transport`, +`subst.force_refinement`) and `ccl_utils`' `PredMemo::rebuild` +(`predicate.rebuild`). + +Three sites record **below** `post_channelize_ir`, so no pane relation folds their +rows: +`simplify` (one combinator covering all thirteen `&mut` rule invocations, with no +rule-body edits), `planning/iterate`, and `transact_phase`'s as-of-read rewrite. +`compile_program` opens no `PassScope` around any of them, so their recordings +are inert in a normal build and land only under an audit a caller opens. +`CAMBRA_LINEAGE_AUDIT=full` (`post-inference..post-as-of-read`) covers the +as-of-read rewrite; reaching `simplify` and `planning/iterate` needs +`CAMBRA_LINEAGE_AUDIT=planning` (`recognized..join-planned`), which is what the +coverage figures below were measured with. `lambda_elim` records nothing, which +is what an audit's endpoint stops in front of. + +### Where to open a recording + +> **Name the node the product replaces, not the pass. One recording per rewrite, +> and split until every product has one.** + +Every pass instrumented so far reduces to one of two forms. + +**Rule-table pass → one wrapper combinator.** `simplify::ruled` wraps all +thirteen rule invocations, with **no rule body edited at all**: + +```rust +fn ruled(label: RewriteLabel, expr: &mut Expr, rule: impl FnOnce(&mut Expr) -> bool) -> bool { + let _g = lineage::enter(expr.node_id(), label, Nature::Machinery); + rule(expr) +} +``` + +This works because a recording declares nothing, so wrapping every *attempt* +rather than every *firing* costs one push and pop when the rule declines. The +`bool` is never consulted: nothing needs to know whether the rule fired. + +**Recursive traversal → one recording at each traversal entry.** +`planning::iterate` names the marker site it rewrites, and `transact_phase`'s +fourteen recordings are one per rewrite it performs. A recording takes only an +**id**, so a site that has already moved `expr.node` out can still open one — +read `expr.node_id()` before the destructure. + +Three refinements the shapes above do not cover: + +- **A product spanning several nodes** — the transaction carrier is what a set of + scattered `with begin():` blocks and register declarations collectively became. + Parent it on the **outermost** node it replaces, never on a synthetic stand-in. +- **The named node may be one the pass itself just minted**, provided it was + minted under a recording: it is then produced by a pass the fold reads, and the fold reaches + the original in two hops. Reading `slot_id` *before* the rewrite does not + require it to be an input-pane node. +- **One entry serving arms of different `Nature`** — open a **second recording on + the same node** inside the arm. The two write disjoint sets of rows on one + parent, which is what two rewrites attributed to one node should look like. A + recording carries one label and one nature for its whole extent by design. + +**And a caution about what the gate can tell you.** `Unexplained == 0` is a +*coverage* property — was anything recording when a node was minted — not a +statement about where the recording pointed. Measured: replacing a pass's +carefully placed recordings with a **single** `enter` on the program root scores +identically on every leak class. Sites are placed correctly because the +attribution *is* the product; the number will not tell you when they are wrong. + +`compile_program` opens one `PassScope` per pass — `Mono`, then `Inline`, +`Transact`, `Letrec`, `Channelize` — inside the single `TableSession` that spans the +whole compile, and retains the drained table as +`CompiledProgram::lineage_table`. A pass that rewrites nothing on a given program +writes no rows, which is the preserve case and not a gap. + +### The node table (`src/ccl/lineage.rs`) + +The recording, keyed by the node it describes. `LineageTable` is +`NodeId → {parents, blame, rule}`: one row per **produced** node, written by +`OpenStep::flush_into_table` as each guard drops. `parents` are the ids the +rewrite consumed (the node the recording named; a fusion's whole consumed set), +`blame` the second edge kind, and `rule` an interned `RewriteTag` — the `{via, +nature, label}` triple is one value because the recording site and its enclosing +`PassScope` settle all three before a row is written, and there are on the order +of fifty distinct triples in the compiler. + +Four properties, each load-bearing: + +- **No `span` column.** Spans are derived by walking `parents` back to a node + the lowering projection covers, which is a handful of hops and does not + lengthen as programs grow. Storing spans per row would denormalize that onto + every row ever minted. +- **"No row" is a legitimate state.** The key space is `NodeId`, and a `NodeId` + can be addressed without ever having been recorded: an id minted by a pass that + records nothing, or one whose producer lies outside the passes a fold reads. + Reads answer empty / `None`; nothing panics. +- **Deaths are taken over rows, never over the key space.** `deaths(live)` is + `recorded ∖ live`, and row enumeration is private so it cannot be taken any + other way: a difference over addressed-but-unrecorded ids would report a death + for every id no pass produced. +- **One table per compile, one pass scope per pass.** A row's key is + process-unique, so no pass set is needed to disambiguate it. That is also why + the pass reaches the write as an *ambient* fact: a recording site knows its + `label` and `nature` but not which pass is running, so `PassScope::enter(pass)` + carries it for the scope's extent and the tag is completed from it. + +**There is deliberately no rewrite-kind column.** A 1:1 copy and a many:1 fusion +differ only in a claim about the origins' *fate*, and driver capture already +decided fate is never declared — it is the live-set difference. What is +load-bearing is the parents column's cardinality, which already carries 1:1, +1:many and many:1. A site that finds itself needing to know the kind is asking a +fate question that nothing here answers. + +The backing store is a `HashMap`; the paged, delta-encoded form is a later pure +re-encoding behind the same accessors. + +A row's `via` is what restricts the whole-compile table to one pane relation: an +id produced by a pass the relation does not span is, to that relation, an ordinary +un-produced id. The restriction is load-bearing — at the second relation a +`Mono`-produced input-pane id has a row, and walking through it would resolve +past the pane. + +Predicate interiors are rows like any other. Lowering's projection covers every +id `collect_tree_ids` enumerates, and `PredMemo::rebuild` records a derived +predicate against the one it was built from. What is **not** recorded is planning +raising a predicate back into the main tree; see +[Known prerequisites for panes past `post-channelize`](#known-prerequisites-for-panes-past-post-channelize). ### The collapse -`collapse(logs, input_ids, output_ids, upstream_attr)` folds a set of logs once, -in pass order, into: +`collapse(table, passes, input_ids, output_ids, upstream_attr)` folds the rows +those passes wrote into: - a `LineageMap` — a dense bidirectional node↔node relation - (self-edge for every survivor), and + (self-edge for every survivor), each edge carrying the **label set** described + in [The edge labels](#the-edge-labels), and - the output pane's `SourceProjection` (`NodeId → SourceAttribution`, where an attribution is `{ spans, rewritten: RewriteTag{via, nature, label} }`). The `rewritten` tag is **mandatory**: a direct image is @@ -228,39 +536,114 @@ in pass order, into: validators carry a debug guard that a `"source"` nature never actually ships. Transients (born + consumed within the phase) compose away. A two-sided leak -audit (`Leak`) guarantees no node silently loses its history: an output with no -lineage (`Unexplained`) and an input that vanished unconsumed (`Dropped`). - -Both checks enumerate from the **tree**. There is no third check on the *produced* -side — "every id a step claims to produce is held by some node" — because -legitimate shapes violate it. - -Uncurrying `def f(x, y)` +audit (`Leak`) reports both sides of the live-set difference: an output with no +lineage (`Unexplained` — a capture defect) and an input absent from the output +pane (`Died` — the death report, data rather than a defect, since under driver +capture nothing declares a fate). + +#### The edge labels + +An edge carries a **set** of labels, not one, and the set is stored once per +`(upstream, downstream)` pair: a pair that is both ancestry and blame is one +edge asserting both, never two edges disagreeing about one pair. Two labels +exist, one per column — `parents` contributes *descends from*, `blame` +contributes *related to, but not consumed* — and a row naming one id in both +columns contributes a single hop carrying both. + +Both are closed transitively, in the same sweep, and the closure composes +**weakest link**: a path is ancestry only while *every* hop on it is an ancestry +hop, and one blame hop anywhere makes the whole path blame. That is what +makes the label mean something at distance — you do not descend from something +you are merely related to two hops back — and it is why the sweep carries a label +alongside each root rather than running two closures whose results could not be +recombined afterwards. Paths meeting at one root union their labels, which is the +other way a pair comes to carry both. + +The dense self-edge is **ancestry**: a surviving node descends from itself, which +is the identity of the weakest-link composition, so density needs no special +case. + +The leak audit reads the ancestry label alone. `ParentUnknown` is a claim about +`parents` — an ancestry hop stopping at an id that describes nothing — while a +blamed id the fold never heard of contributes no edge and no class, the same +silence attribution keeps for a blamed id with no known spans. + +#### The fold is order-free + +`collapse` reads the rows as an **edge set**: a row contributes a hop `p → x` for +every `p ∈ parents(x) ∪ blame(x)`, labelled by the column(s) that named it, and +`roots(x)` maps each `u ∈ input_ids` with `u ⇝ x` to the label of the paths that +reach it. A node's annotation lives in the commutative monoid of labelled root +maps under union, which together with one row per id is what makes the result +independent of the order the rows were written in. + +That is not a nicety: write order is **not** chronology, since rows are written +when their guard drops, so an enclosing rewrite's rows land after the rows of the +rewrites nested inside it. + +Two invariants make the fold a single ascending-`NodeId` sweep — no fixed point, +no memoisation, no cycle guard: + +- **one row per id**, and **no node is its own parent**, so no edge runs + backwards out of a self-referential definition; +- **monotone minting** — `NodeId`s come from one process-global counter and a + row's node is *captured* (via `on_mint`) after its upstream ids were read, so + every id a row names is older than the node naming it — both columns alike + (debug-asserted at the sweep). + +Together, ascending `NodeId` *is* a topological order of the definition graph. +`sweep_metrics` measures the falsifier: a non-zero backward-edge count is exactly +the number of vertices a sweep would have to revisit. + +The leak taxonomy follows the set reading, and has three classes. `Unexplained` +is an output-pane id no row produced and the input pane does not hold. `Died` is +the input-pane set difference. `ParentUnknown` is a parent that is neither an +input-pane id nor produced by a pass the fold reads — **one** class for both edge +shapes, because a lone parent and one of a fusion's several are the identical +condition, and telling them apart would mean recording the rewrite's shape. + +The classes that are properties of a *record* rather than of a fold live at +the write instead: one row per id, and every row anchored through consumption or +blame, are asserted in `LineageTable::record` (see +[The lineage model](#the-lineage-model-srcccllineagers)). "Two rewrites claimed +one id's death" has no class at all — a fate claim is not something a row makes, +and an id appearing in two rows' `parents` is an ordinary shared ancestor. + +`collapse_lowering` is **sequential**, and is the one fold that should be: its +leaf entries are appended at construction rather than when a guard drops, so its +log +genuinely is chronology, and its last-tag-wins re-imaging (`lower_expr` re-tagging +an arm's already-tagged root) is real semantics rather than an artifact. It also +has no lineage to compose — lowering mints from scratch, so what would be a set +of input-pane roots per id degenerates to a plain live set. + +Both checks enumerate from the **tree**. There is deliberately no third check on +the *produced* side — "every recorded id is held by some node" — because two +legitimate shapes would read as violations. Uncurrying `def f(x, y)` builds one `__arg_tuple_0.0` projection template and substitutes a freshened copy of it at each `x`; every copy's root carries that occurrence's own id, so the template's own root id is tagged and then held by no node. The read-your-writes values the mutability phases keep in a substitution `env` work the same way. An -id retired like that leaves no trace — a *consumed* id shows up in the fold as -absence from `roots`, but a replaced one looks exactly like a live node the check -cannot see. Saying it explicitly means emitting the discard the model already -has, `Transform { consumed: [id], produced: [] }`, which neither site does today. +id retired like that leaves no trace: it looks exactly like a live node the check +cannot see. Construction closes the gap the check would have watched: a node is built either -by `Expr::new` (mint, recorded) or `Expr::preserve` (carry an existing +by `TypedExpr::new` (mint, recorded) or `TypedExpr::preserve` (carry an existing id, nothing recorded), so an id cannot be minted and then discarded. -The fold is in tree; **planned** is its use at the inspector's two pane -boundaries, which needs the pass logs above. +The fold runs over the inspector's two pane relations today; **planned** is the +inspector's consumption of what it produces. ## The seam (`src/ccl/context.rs`) - **The lowering log + fold.** Lowering records a `LoweringLog` under an - always-on `RecorderSession::lowering()` (installed in every build across - `lower_stmts`, drained before the first pass session). It records at **leaf - grain**: `tag_source` / `tag_image` / `tag_machinery` are thin shims appending - a single-node leaf `LoweringStep` (`Transform { consumed: [], produced: [id] }`, - anchored at the nearest real span). Ordinary mints open **no** frame, so - `on_mint` stays a no-op on the hot path; frames open only where ambient `Copy` + always-on `LoweringSession` (installed in every build across `lower_stmts`, + drained before the first pass scope opens). It records at **leaf grain**: + `tag_source` / `tag_image` / `tag_machinery` are thin shims appending a + `LoweringStep::Leaf` (one id, anchored at the nearest real span). Ordinary + mints record **nothing**, so + `on_mint` stays a no-op on the hot path; a recording opens only where ambient + `Copy` capture is needed — uncurry's template discharge and the chained-comparison operand freshens, whose interior re-mints land as `Copy` LoweringSteps mirroring their origins (`copy_frame`, which declares that shape and so has no @@ -302,42 +685,85 @@ boundaries, which needs the pass logs above. consumption shows which distinction is load-bearing. Treat both axes as unstable until then. - At the lowering→pipeline handoff (before uniquify and inference, so the release + At the lowering→pipeline handoff (before uniquify/inference, so the release `InferError` read timing is unchanged) `collapse_lowering` folds the log **once** into the always-on **lowering projection** (`NodeId → - SourceAttribution`, covering every id `collect_tree_ids` enumerates, refinement - predicates included). This is the degenerate lowering case of `collapse` - (shared `RootTracker` core): no input pane (roots start empty, leaves are pure + SourceAttribution`, covering every id `collect_tree_ids` enumerates, + refinement-predicate interiors included). This is the degenerate lowering case + of `collapse`, + and the one fold that stays sequential: no input pane (leaves are pure insertions attributed from their literal anchor), no `LineageMap` output, no - upstream attr (a `Copy` mirrors its origin's already-folded entry). `Pass::Lower` - is lowering-projection vocabulary only — it never appears in `pass_lineage` or - the inter-pane `RewriteStep` logs, which stay homogeneous `NodeId → NodeId` - (the lowering projection constrains the **product**, not the producer). + upstream attr (a copy mirrors its origin's already-folded entry), and no + one-record-per-id requirement (a re-image is a second entry for one id and the + later tag deliberately wins). `Pass::Lower` is lowering-projection vocabulary + only — it never tags a `LineageTable` row, and the inter-pane relation stays a + homogeneous `NodeId → NodeId` (the lowering projection constrains the + **product**, not the producer). Release `InferError` diagnostics read the projection one-hop (no fold, before any pane exists). `CompiledProgram` retains it as `lowering_projection`. - **The lowering leak gate.** `collapse_lowering`'s leak taxonomy replaces the retired `assert_seed_coverage` (and its `SEED_EMPTY_SPANS_ALLOWLIST`): an unrecorded lowering mint surfaces as `Leak::Unexplained` (every output-tree - node must be explained by a leaf or a copy); a born-copied-discarded template - id composes away (live but neither placed nor an output). The fold is always-on + node must be explained by a leaf or a copy), and a copy of an origin no earlier + record covered as `Leak::ParentUnknown`; a born-copied-discarded template id + composes away (live but neither placed nor an output). The fold is always-on (its product is release-critical); the leak *checks* are debug/test-gated at - the boundary via `assert_leaks_clean`. Orphaned projection keys are + the lowering handoff via `gate_leaks`. Orphaned projection keys are structurally impossible — the projection is *produced by* the fold, never mutated incrementally. -- **Planned — materialization (cold, inspector-only).** `CompiledProgram::materialize_panes` - folds `pass_lineage` at the two pane boundaries: the Mono log bridges - pre → post-inference; the Inline + Transact + Letrec + Desugar logs bridge - post-inference → post-desugar. It returns the three per-pane - `SourceProjection`s and the two pane-pair `LineageMap`s. **Both boundaries are - fully recorded** — every pass emits its mints/consumes/copies as `RewriteStep`s - — so there is no catch-all bridge — every node is explained by a recorded step. -- **Planned — the pane leak gate.** `assert_leaks_clean` is in tree and gates the - lowering boundary today; planned is applying it at both **pane** boundaries, - which assert **zero** leaks of every class: an `Unexplained` (uncaptured mint) - or a `Dropped` (unconsumed vanishing node) is a recording bug, not tolerated - residue. With the - full-coverage lowering projection, `unresolved` projection entries are a hard - zero at every pane too — to be asserted structurally by the census ratchet. +- **Materialization (cold, inspector-only).** `CompiledProgram::materialize_panes` + folds `lineage_table` across the two pane relations, restricting it by pass: + `MONO_WINDOW` bridges pre → post-inference; `CHANNELIZE_PASSES` (Inline, Transact, + Letrec, Channelize) bridges post-inference → post-channelize. It returns the three per-pane + `SourceProjection`s, the two pane-pair `LineageMap`s, and each relation's leak + vector. There is no catch-all bridge: a node is explained by a recorded row or + it is not explained at all, and the gate is what says which. Materialization + cannot assert its own gate — with `Died` as a payload the leak vector is a + *product*, not an error channel — so it returns the leaks and callers gate. +- **The pane leak gate.** `gate_leaks` gates the lowering handoff and both + **pane relations** on the *defect* classes: `Unexplained` (an output node no + capture explains) and `ParentUnknown` (a lineage edge to an id the fold has + never heard of). The split lives on `Leak::is_defect`. `Died` is excluded by + construction — it is the death report, and gating on it would be unsatisfiable + now that no pass declares what it consumes. + + Where the gate stands on a real corpus: **zero on every gated class, at both + pane relations, for every program.** Capture is total over the adopted span — + every output-pane node has an origin — and the corpus test asserts it as a + property rather than pinning a residue count. + +## Known prerequisites for panes past `post-channelize` + +A pane may be issued at **any** point during compilation — the current adoption +point is an artifact of what has been built, not a statement about the design. +Three things block extending it, all acknowledged and none blocking the two panes +that exist: + +- **`lambda_elim` records nothing, and re-mints** nearly every pass-through + node, so a pane pair spanning it would have no id correspondence to join on. + Its catch-all traversal arm carries a `TODO(preserve)`, and `planning/groupby` + relies on that re-minting to launder predicate-interior ids it lifts out of a + type — `groupby_recognition_lifts_the_key_without_aliasing` pins the reliance — + so a preserve there owes `groupby` an explicit freshen. +- **Operator conversion has no identity.** `TileOperator` carries none and there + is no `OperatorId`, so a pane after it has nothing to resolve against. +- **Planning does not record what it raises out of the predicate domain.** Three + sites cross: `planning/iterate`'s `fn_of_bare_predicate` lift, the group-by key + extraction, and the hash-join key morphisms. Each would record against its + term-tree site, that being the node the raised material becomes, but the + `on_copy` hook reports the *origin it freshened*, and no channel re-roots a + captured copy onto the node the site named. Only `planning/iterate` records at + all, and no pass scope covers it. + + Measured at `post-inference..join-planned`, when that was `full`'s span: over + the 11-program corpus the residue was 1184 `ParentUnknown` edges and nothing + else, every unknown parent a predicate-interior id of the input tree, and + widening the audit's live set (`CAMBRA_LINEAGE_PREDICATES=1`) took every gated + class to zero. (The count is from before the endpoint moved and has not been + re-taken.) That says the recording is total for the span and the narrow live + set is what makes the crossing read as a leak. It says nothing about the two + pane relations, whose passes rebuild predicates through `PredMemo` and are + recorded there. ## Planned — inspector consumers (`src/inspector_model/`) @@ -355,8 +781,9 @@ pane folds; the release compiler reads the lowering projection and nothing else. string; the frontend formats the tag itself. - `paneLinks` ship each pane-pair `LineageMap` **dense** — self-edges included, no identity-edge filter — via `stage::dense_edges`; the frontend only follows - edges, never reconstructing them. Both validators check that every edge - endpoint is a live node id in its respective pane. + edges, never reconstructing them, and reads each edge's label set to decide + whether to render the blame or prune it. Both validators check that every + edge endpoint is a live node id in its respective pane. The snapshot payload carries its own version in `meta.schema`, owned by the inspector crate along with the fixture corpus pinned to it. Nothing under diff --git a/src/ccl/design/type-inference.md b/src/ccl/design/type-inference.md index 37d5b6f5..c094119e 100644 --- a/src/ccl/design/type-inference.md +++ b/src/ccl/design/type-inference.md @@ -573,8 +573,9 @@ anything in planning is **unmeasured** — the superlinearity argument above com from the original split's shape, not this one — and that measurement is what decides between fixing the producer (memoize the rebuild, or keep the origin `Rc` when the freshen is vacuous) and narrowing the invariant to "preserved through -inference, deliberately re-split at instantiation". Tracked in the -lineage-redesign doc, §12.4(9). +inference, deliberately re-split at instantiation". Tracked as an open decision +in the `lineage-design` note under projects/program-inspector in the internal +vault. A pass reaches predicates through **every type slot a node carries**, not just `expr.ty`: the node's own type, its `user_annotation`, a `Cast`'s `target`, and — @@ -760,11 +761,11 @@ For the reconcile to hold, the passes that *introduce* refined types post-infere #### Feed handles as an invariant `History` constructor (`Type::History { kind: Feed }`) -A feed handle is `Type::History { value: 𝑇, domain: 𝐷, kind: HistoryKind::Append }` (displayed `feed(𝐷 ⇒ 𝑉)`) — a function `𝐷 ⇒ 𝑇` carried as two children plus a two-valued `kind` marker. It **shares the `Type::History` variant with a mutable variable** (`kind: Overwrite`, displayed `Mut(𝑉, 𝐷)`); the two were unified from the former `Type::Feed(ρ)` / `Type::Mut{…}` pair (see [`Mut` is a CCL type](mutability.md#mut-is-a-ccl-type)). `let 𝑑 = Defer in body` gives `𝑑` a `Feed`-kind history whose channel `𝐷 ⇒ 𝑇` is the *post-desugar result type* of the binding (a `𝐷 ⇒ 𝑇` channel for fed defers, the defined value's type for `<<=`-defined defers). Like `Hole` and `Infer` the `Feed` kind is **transient**, scoped to inference: `channelize` (which runs after inference) eliminates every defer construct along with its feed histories, and no pass downstream of it may observe one. (This is the feed-handle type of [`Feed` is a CCL type](mutability.md#feed-is-a-ccl-type) — what a defer-mediating UDF parameter carries.) +A feed handle is `Type::History { value: 𝑇, domain: 𝐷, kind: HistoryKind::Append }` (displayed `feed(𝐷 ⇒ 𝑉)`) — a function `𝐷 ⇒ 𝑇` carried as two children plus a two-valued `kind` marker. It **shares the `Type::History` variant with a mutable variable** (`kind: Overwrite`, displayed `Mut(𝑉, 𝐷)`); the two were unified from the former `Type::Feed(ρ)` / `Type::Mut{…}` pair (see [`Mut` is a CCL type](mutability.md#mut-is-a-ccl-type)). `let 𝑑 = Defer in body` gives `𝑑` a `Feed`-kind history whose channel `𝐷 ⇒ 𝑇` is the *post-channelize result type* of the binding (a `𝐷 ⇒ 𝑇` channel for fed defers, the defined value's type for `<<=`-defined defers). Like `Hole` and `Infer` the `Feed` kind is **transient**, scoped to inference: `channelize` (which runs after inference) eliminates every defer construct along with its feed histories, and no pass downstream of it may observe one. (This is the feed-handle type of [`Feed` is a CCL type](mutability.md#feed-is-a-ccl-type) — what a defer-mediating UDF parameter carries.) Below, **`Feed(ρ)`** abbreviates a `kind: Feed` history whose reconstructed channel is `ρ = 𝐷 ⇒ 𝑇`; the `value`/`domain` children are the two halves of `ρ`. An `Overwrite` history reaches the relation as a handle — a read has already dereffed at the rule that emitted it — so the four invariance rules below are specifically the `Feed`-kind behavior. -The typing rules (`infer_simple_sub::emit_defer` / `emit_feed` / `emit_define`): `Defer` emits `Feed(fresh ρ)`; `Feed{name, value}` and `Define{name, value}` type as `Unit`, resolve `name` from the scope like a `Var` use, and constrain their contribution into the target's payload (`Fun(fresh δ, value_ty)` for a feed — the channel *domain* is a desugar artifact, so `δ` stays unconstrained and coalesces to `Infer`; the bare `value_ty` for a define). A target that isn't structurally a feed handle (a lambda parameter — ParamAsTarget) is demanded to be one via the upper bound `target <: Feed(ρf)`; the call-site argument edge meets it there and invariance carries the contribution back to the caller's channel. A bare `Defer` RHS is never generalized (`should_generalize` wants a lambda RHS), so feeds and reads of one defer share one `ρ`; a defer minted inside a generalized function instantiates fresh per call site. +The typing rules (`infer_simple_sub::emit_defer` / `emit_feed` / `emit_define`): `Defer` emits `Feed(fresh ρ)`; `Feed{name, value}` and `Define{name, value}` type as `Unit`, resolve `name` from the scope like a `Var` use, and constrain their contribution into the target's payload (`Fun(fresh δ, value_ty)` for a feed — the channel *domain* is a channelize artifact, so `δ` stays unconstrained and coalesces to `Infer`; the bare `value_ty` for a define). A target that isn't structurally a feed handle (a lambda parameter — ParamAsTarget) is demanded to be one via the upper bound `target <: Feed(ρf)`; the call-site argument edge meets it there and invariance carries the contribution back to the caller's channel. A bare `Defer` RHS is never generalized (`should_generalize` wants a lambda RHS), so feeds and reads of one defer share one `ρ`; a defer minted inside a generalized function instantiates fresh per call site. `History` is the lattice's only **invariant** constructor. Feeding is a contravariant capability (a feed contributes an element *into* the channel) while reading is covariant, so a feed handle flowing through a function parameter must propagate feed contributions *backwards* to the caller's channel — a one-way `arg <: param` edge would strand the callee's contribution on the parameter variable. Four constraint rules (`constrain_go`), where `Feed(a)`/`Feed(b)` are same-`kind` (`Feed`) histories: @@ -920,7 +921,7 @@ Some refinement predicates **close over an outer binder**. The motivating case i **Pi types.** `Type::Fun` carries an optional binder: `Fun { name: Option, domain, codomain }`. `name: Some(𝑥)` is the dependent type `(𝑥: domain) ⇒ codomain`, with `𝑥` bound in `codomain`; `name: None` is the ordinary function type. `emit_lambda` always names the binder from the lambda parameter, so a predicate that closes over the parameter stays bound. The binder is **cosmetic for ordinary functions** — `coalesce_compact_go` keeps it only when the codomain's refinement predicates actually reference it (queried via `subst::type_free_vars`) and strips it otherwise, so monomorphic output is unchanged and equality/printing don't churn. -**Substitutions and contexts (`ccl::subst`).** A `Subst` is a context morphism that maps *term* binders (`Var` names) to replacement `TypedExpr`s. It never relabels a type variable — that is freshening's job. Two flavours: a **rename** `[𝑘 ↦ 𝑥]` (invertible) and a **discharge** `[𝑥 ↦ arg]` (one-way). The traversal is uniform over terms and types: `apply_expr` rewrites each node's type slots via `apply_type` in the same pass, so a substituted binder occurring inside a type-borne refinement predicate is discharged where it sits (no value-only contract, no dangling residual for §6.2 to catch in release builds). It is a true no-op when no substituted binder occurs free in the term — value or type slots — so a vacuous discharge from a non-dependent application changes nothing and shares the predicate `Rc`. Capture is impossible under the Barendregt convention (binder uids are minted once at lowering; copies preserve them) and the engine *asserts* it instead of α-renaming. Predicates are immutable, so a substitution always *rebuilds* a changed predicate (a fresh `Rc`); the engine drives two modes that differ only in what else they touch: **transport** (`apply_expr`/`apply_type`, builds new terms — the constraint-edge flavour) and **in-place rewrite** (`rewrite_expr`, mutates the term tree the caller owns; a predicate the substitution actually touches is rebuilt, one it merely walks past keeps its `Rc` — the pass-level flavour that `lambda_elim::substitute`, `channelize::desugar_substitute`, inlining's beta step, and lowering's uncurrying all wrap). Both modes thread the same `PredMemo`, so occurrences that shared one term are re-pointed at the same result. A **context** (`well_formed` / `type_free_vars`) is the dual *checking* device: a type is well-formed iff its predicates' free term-vars are in scope. +**Substitutions and contexts (`ccl::subst`).** A `Subst` is a context morphism that maps *term* binders (`Var` names) to replacement `TypedExpr`s. It never relabels a type variable — that is freshening's job. Two flavours: a **rename** `[𝑘 ↦ 𝑥]` (invertible) and a **discharge** `[𝑥 ↦ arg]` (one-way). The traversal is uniform over terms and types: `apply_expr` rewrites each node's type slots via `apply_type` in the same pass, so a substituted binder occurring inside a type-borne refinement predicate is discharged where it sits (no value-only contract, no dangling residual for §6.2 to catch in release builds). It is a true no-op when no substituted binder occurs free in the term — value or type slots — so a vacuous discharge from a non-dependent application changes nothing and shares the predicate `Rc`. Capture is impossible under the Barendregt convention (binder uids are minted once at lowering; copies preserve them) and the engine *asserts* it instead of α-renaming. Predicates are immutable, so a substitution always *rebuilds* a changed predicate (a fresh `Rc`); the engine drives two modes that differ only in what else they touch: **transport** (`apply_expr`/`apply_type`, builds new terms — the constraint-edge flavour) and **in-place rewrite** (`rewrite_expr`, mutates the term tree the caller owns; a predicate the substitution actually touches is rebuilt, one it merely walks past keeps its `Rc` — the pass-level flavour that `lambda_elim::substitute`, `channelize::channelize_substitute`, inlining's beta step, and lowering's uncurrying all wrap). Both modes thread the same `PredMemo`, so occurrences that shared one term are re-pointed at the same result. A **context** (`well_formed` / `type_free_vars`) is the dual *checking* device: a type is well-formed iff its predicates' free term-vars are in scope. **Edges carry substitutions, stored two-sided in their native direction.** Each entry of a variable's bound lists is a `Bound { self_subst, ty, ty_subst }`: an upper entry on `𝑉` reads `𝑉‹self_subst› <: ty‹ty_subst›`, a lower entry `ty‹ty_subst› <: 𝑉‹self_subst›` (both identity for ordinary bounds). `constrain_subtype` delegates to `constrain_go(lhs, rhs, sl, sr, cache)` — each side under its own morphism. The **Fun/Fun arm derives the binder correspondence** `[𝑘 ↦ 𝑥]` onto the lhs side of the codomain edge, and the contravariant domain edge **swaps the two sides** rather than inverting anything. The var arms record edges verbatim — *nothing is inverted at record time*. A **discharge has no inverse**, so edges are recorded in their native direction rather than pre-inverted and re-inverted during closure (which would degrade a discharge to the identity, silently destroying it whenever a consumer edge is recorded before the producer's concrete codomain arrives — the opaque/higher-order application order, O3). Under identity morphisms every arm reduces exactly to the substitution-free solver, so all monomorphic inference is byte-identical. diff --git a/src/ccl/expr.rs b/src/ccl/expr.rs index 08d6ee6d..ca247f18 100644 --- a/src/ccl/expr.rs +++ b/src/ccl/expr.rs @@ -723,9 +723,14 @@ pub struct TypedExpr { /// (see [`crate::ccl::provenance`]). Excluded from [`PartialEq`] because /// provenance is metadata, not part of the node's value. /// - /// **`Clone` freshens** (see the [`Clone`] impl below), so reaching a - /// duplicated id takes writing one deliberately, through - /// [`preserve`](Self::preserve). + /// **`Clone` freshens.** A clone is a *sibling*, not the same node, so the + /// hand-written [`Clone`] impl below mints a new id for every node it + /// copies and reports each `(origin, fresh)` pair to the lineage recorder. + /// There is no decision to make at a clone site and no "copy then freshen" + /// step to forget: the only way to reach a duplicated id is to write one + /// deliberately, through [`preserve`](Self::preserve), + /// [`clone_at`](Self::clone_at), or + /// [`clone_preserving_ids`](Self::clone_preserving_ids). /// /// # What is forbidden is a mint, not a write /// @@ -756,34 +761,29 @@ pub type Expr = TypedExpr; /// is reported to the ambient lineage recorder via /// [`on_copy`](crate::ccl::lineage::on_copy). /// -/// A derived `Clone` would copy `node_id`, making every duplication site decide -/// whether to keep the id or freshen it. Freshening here removes the decision; -/// `src/ccl/design/provenance.md`, "Node identity (`src/ccl/provenance.rs`)" has -/// what a wrong decision costs. +/// A derived `Clone` copied `node_id`, which made *every* duplication site a +/// decision: reach for `clone` when the copy replaces its source, or for a +/// `fresh_copy` helper when both reach the output tree. Get it wrong and two +/// nodes share an id — which collapses them into one entry in every +/// `NodeId`-keyed walk, makes the pane projection ambiguous (one attribution for +/// two nodes), and is caught, if at all, by an id-uniqueness assert far from the +/// site. Freshening here removes the decision: sharing an id now requires +/// writing one deliberately, through [`TypedExpr::preserve`], +/// [`TypedExpr::clone_at`], or [`TypedExpr::clone_preserving_ids`]. /// -/// **The named id-sharing paths.** Sharing an id takes writing one through -/// [`TypedExpr::preserve`] (one node at an id already in hand), -/// [`TypedExpr::clone_preserving_ids`] (a subtree at its source's ids), the -/// [`preserving_ids`](crate::ccl::lineage::preserving_ids) scope that backs it — -/// called directly by `PredMemo`'s rebuilds in `crate::ccl::ccl_utils` — the -/// `*_preserving` constructors -/// ([`expr_stmt_preserving`](TypedExpr::expr_stmt_preserving), -/// [`let_in_preserving`](TypedExpr::let_in_preserving)), which are `preserve` in -/// convenience form, or the one literal in [`crate::ccl::subst`]'s -/// `as_expr_preserving`, where a substituted occurrence's id lands on the -/// replacement's root. -/// -/// The freshen is deep by construction: `node.clone()` clones the children, and -/// each child is a `TypedExpr` reaching this same impl. +/// **The recursion is the derive's.** `node.clone()` clones the children, and +/// each child is a `TypedExpr` reaching this same impl — so the freshen is deep +/// by construction and fused into the copy, one walk rather than the copy-then- +/// freshen pair it replaces. /// /// **Type slots are not freshened, and that is the rule, not an omission.** A /// [`Type`] carries no identity: the only [`NodeId`]s reachable through one are /// the `TypedExpr`s inside a `Refinement.predicate`, which is an /// `Rc` — so `ty.clone()` bumps a refcount and reaches this impl not /// at all. That is load-bearing twice over: predicate interiors are outside the -/// *uniqueness* domain (`assert_unique_node_ids` walks children only), and -/// planning's compile memo is keyed on `Rc` identity, so splitting the sharing -/// would compile one predicate once per copy. +/// id domain (`assert_unique_node_ids` excludes them deliberately), and planning's +/// compile memo is keyed on `Rc` identity, so splitting the sharing would compile +/// one predicate once per copy. /// /// **`NodeId::PLACEHOLDER` is not preserved.** A [`throwaway`](TypedExpr::throwaway) /// node is built to be rendered into a panic message, never cloned into a tree; @@ -854,8 +854,11 @@ impl TypedExpr { /// `src` is some *other* node — is this constructor's shape, and the one where /// a stray `node_id: NodeId::fresh()` hides: a preserve and a mint differ by /// one token in otherwise identical five-line literals. Those sites are marked - /// `TODO(preserve)` and are greppable; five remain, in `channelize`, - /// `mut_elim`, and `transact_phase`. + /// `TODO(preserve)` and are greppable: thirteen across five files — + /// `channelize` (eight), `mut_elim` (two), and one each in `transact_phase`, + /// `subst`, and `lambda_elim`. Eleven are this reach-for-another-id shape; the + /// two in `subst` and `lambda_elim` ask a different question, whether their + /// rebuild should mint or preserve at all. /// /// **A field-wise rebuild** — `let TypedExpr { node, ty, user_annotation, /// node_id } = expr;` then rebuilding with one child swapped — is *not* this @@ -897,55 +900,77 @@ impl TypedExpr { self.node_id } - /// A deep copy at the **same identities** — the opt-out from the freshening - /// [`Clone`], and the subtree analogue of [`preserve`](Self::preserve). - /// - /// Discouraged, and narrow: three shapes call it. Anywhere else a copy that - /// duplicates ids is a bug an id-uniqueness assert will find later, and the fix - /// is to **record** the freshened copy rather than to suppress the freshen — - /// including when the symptom is a `Leak::Unexplained` or a - /// `Leak::CopyOfUnknown`, which mean a copy was made with no step open or - /// against an unrecorded origin. Freshening everywhere and recording it costs - /// no compile time and no meaningful memory, so the fix is at the copy site. - /// See the vault's `freshening-clone-report`. - /// - /// # 1. A `Subst` discharge template + /// A deep copy at the **same identities** — the subtree analogue of + /// [`preserve`](Self::preserve), and the opt-out from the freshening + /// [`Clone`]. /// - /// A [`Subst`](crate::ccl::subst::Subst) discharge payload is never a tree - /// node. `Mapping::as_expr` clones it afresh at every read, and that read is - /// where each sibling is minted, so copying the template itself must mint - /// nothing. Every site of this shape is either the argument to - /// `Subst::discharge` or `Mapping`'s own `Clone` propagating one. + /// Reach for this in exactly two situations. Anywhere else, a copy that + /// duplicates ids is a bug waiting to be found by an id-uniqueness assert, + /// and the right fix is to **record** the freshened copy — open a recording + /// around it — not to suppress the freshen. /// - /// # 2. A throwaway copy + /// # 1. A snapshot taken for rollback or comparison /// /// A copy the normal path *discards*, kept only so a failure or a later - /// comparison has something to look at: lowering's per-statement rollback copy - /// of the accumulated continuation (`lower_stmts_recovering`) and the - /// post-inference type check's scratch tree (`infer::check::check`). - /// Freshening them would mint whole trees for values nothing reads — - /// quadratic in both cases; each site carries a `TODO` saying so, and the fix - /// at both is to stop needing the copy at all. + /// comparison has something to look at. Lowering's per-statement rollback + /// copy of the accumulated continuation (`lower_stmts_recovering`) and the + /// post-inference type check's scratch tree (`infer::check::check`) are both + /// this shape. Freshening them would mint whole trees for values nothing + /// reads — quadratic in both cases; each site carries a `TODO` saying so, and + /// the real fix at both is to stop needing the copy at all. /// - /// A copy that *reaches the output* is not this shape, even when the source is - /// dropped on the way: the output copy is a sibling and freshens. - /// - /// # 3. A test comparing trees across a pass + /// # 2. Tests that compare trees across a transformation /// /// A test that runs a pass over a copy and compares against the original - /// needs the two to be the same nodes, or it is not testing the pass. See + /// needs the two to be *the same nodes*, or it is not testing the pass. See /// `uniquify`'s idempotence and id-stability tests. /// /// # Why this is sound /// - /// In no shape are the source and the copy both reachable from one tree, so - /// nothing ever observes two live nodes at one identity. A template is not a - /// tree node; a throwaway sits outside the tree the pipeline goes on rewriting. + /// In both shapes the copy **replaces or shadows** its source rather than + /// standing beside it: at no point are both reachable from one tree, so + /// nothing ever observes two live nodes at one identity. + /// + /// # What this is *not* for + /// + /// Not for silencing a `Leak::Unexplained` or `Leak::ParentUnknown`. Those + /// mean a copy was made with nothing recording, or against an origin the + /// table never recorded — a **recording** gap. Measured: freshening + /// everywhere and recording it costs no compile time and no meaningful memory + /// (the naive arm ran *faster* than baseline at 2-3x the ids), so the honest + /// fix is to record the copy. See the vault's `freshening-clone-report`. pub(crate) fn clone_preserving_ids(&self) -> Self { let _preserving = crate::ccl::lineage::preserve_ids(); self.clone() } + /// A copy whose **root carries `node_id`** and whose interior is freshened — + /// the root-carry primitive. + /// + /// The substitution engine's compound-replacement arm is the caller: the + /// replacement for a `Var(𝑥)` occurrence denotes what the occurrence denoted + /// — the value of 𝑥 *at that position* — so the occurrence keeps its own id, + /// inheriting its span and attribution, while the interior becomes a fresh + /// node-set. N reads give N distinct roots. + /// + /// The root is built directly at `node_id` rather than minted and then + /// overwritten. That matters now that `Clone` freshens: `clone().re_root(id)` + /// minted a root id, fired `on_copy` for it, and then discarded it — one + /// stranded id and one stranded row per substituted occurrence. Here nothing + /// is minted for the root at all. + /// + /// The interior still freshens, because `node.clone()` reaches each child's + /// own [`Clone`], so each child is a sibling of the template's and records as + /// one. + pub(crate) fn clone_at(&self, node_id: NodeId) -> Self { + TypedExpr { + ty: self.ty.clone(), + node: self.node.clone(), + user_annotation: self.user_annotation.clone(), + node_id, + } + } + /// Set the inferred type on this expression, consuming and returning it. /// /// Used to pre-fill the type in tests or when the type is known at construction time. diff --git a/src/ccl/infer/api.rs b/src/ccl/infer/api.rs index 48422c29..356a375e 100644 --- a/src/ccl/infer/api.rs +++ b/src/ccl/infer/api.rs @@ -512,7 +512,7 @@ pub enum InferError { /// first (`k = x`) does not help, because discharging `[k ↦ x]` puts the mutable variable's /// name straight back into the predicate. Reported here so the program is rejected /// with its source position instead of tripping the debug-only scope net (and, in - /// release, surviving to panic at the pre-desugar wall). + /// release, surviving to panic at the pre-channelize wall). MutableInRefinedType { /// The mutable variable's name. name: String, @@ -883,7 +883,7 @@ impl std::fmt::Debug for InferError { /// tree is fully annotated and contains no `Type::Hole`; defer constructs /// may still carry `Type::History` (feed) types with `Type::Infer` channel domains /// — those are erased by `channelize`, which runs next (see -/// [`Strictness::PreDesugar`]). +/// [`Strictness::PreChannelize`]). /// /// It also **consumes every user annotation**: annotations are an input to /// inference, and on success no `user_annotation` slot survives it (see @@ -1006,7 +1006,7 @@ pub enum Strictness { /// channels (`Feed` histories with a rigid `ChanDom(d)` domain) and /// induction accumulators (`Overwrite` histories whose `Infer` domain the unified /// phase resolves), none of which are erased yet. - PreDesugar, + PreChannelize, } /// Check that every [`crate::ccl::TypedExpr::ty`] and [`crate::ccl::TypedBinding::ty`] @@ -1162,7 +1162,7 @@ fn collect_type_errors( at: context_sym.to_string(), }), Type::Infer(var) => { - // Pre-desugar, an induction accumulator's domain is still `Infer` (a + // Pre-channelize, an induction accumulator's domain is still `Infer` (a // `Mut(V)` with no annotated domain — the unified phase resolves it // to the writing loop's extent); the relaxation tolerates it (see // [`Strictness`]). Feed channel domains are the rigid `ChanDom` @@ -1174,7 +1174,7 @@ fn collect_type_errors( }); } } - // a nominal channel domain is a pre-desugar artifact + // a nominal channel domain is a pre-channelize artifact // exactly like an `Infer` channel domain — `channelize` must // substitute it away; a survivor at the strict wall is a compiler bug. Type::ChanDom(name, _) => { @@ -1303,15 +1303,15 @@ pub fn typecheck(expr: &Expr) -> Result<(), Vec> { /// the same hole-freeness and semantic checks, but transient histories /// (`Feed` channels with a `ChanDom` domain, `Overwrite`s with an `Infer` domain) /// are permitted — the unified phase and `channelize` erase them (see -/// [`Strictness::PreDesugar`]). -pub fn check_pre_desugar(expr: &Expr) -> Result<(), Vec> { +/// [`Strictness::PreChannelize`]). +pub fn check_pre_channelize(expr: &Expr) -> Result<(), Vec> { // The relaxation applies only when the tree carries transient histories // (feed/mutable machinery). A program with none should be fully resolved // after inference, so a residual `Infer` there is an ambiguous program // (e.g. an unexercised generic) — check it strictly so it surfaces as an // `UnresolvedInfer` diagnostic. - let strictness = if has_pre_desugar_artifacts(expr) { - Strictness::PreDesugar + let strictness = if has_pre_channelize_artifacts(expr) { + Strictness::PreChannelize } else { Strictness::Strict }; @@ -1319,19 +1319,19 @@ pub fn check_pre_desugar(expr: &Expr) -> Result<(), Vec> { crate::ccl::infer::check(expr) } -/// Whether the tree carries pre-desugar artifacts: a `Defer`/`Feed`/ +/// Whether the tree carries pre-channelize artifacts: a `Defer`/`Feed`/ /// `Define` node, **or** a transient [`Type::History`] (a feed channel or a /// mutable variable) in any reachable type slot — and hence the channel -/// `Feed`/`ChanDom` (and `Overwrite`-history `Infer`-domain) types the pre-desugar check +/// `Feed`/`ChanDom` (and `Overwrite`-history `Infer`-domain) types the pre-channelize check /// tolerates. /// /// The transient-*type* check matters because a defer-read *alias* (`Var(x) : /// feed(_)`) carries a `Feed` type with no defer *node*. Inline runs before -/// desugar, so its beta-reduction ([`crate::ccl::lambda_elim::substitute`]) +/// channelize, so its beta-reduction ([`crate::ccl::lambda_elim::substitute`]) /// can hand such an alias to [`debug_typecheck`] as a standalone subtree; /// keying only on defer nodes would wrongly check it strictly and reject the /// legitimate channel type. `Mut` is analogous: a mutable reference carries a -/// `Mut` type whose `Infer` domain the pre-desugar relaxation must tolerate +/// `Mut` type whose `Infer` domain the pre-channelize relaxation must tolerate /// until the unified phase resolves it. /// /// The transient type can live on a **binder slot** rather than a node type — a @@ -1340,7 +1340,7 @@ pub fn check_pre_desugar(expr: &Expr) -> Result<(), Vec> { /// the `Let` node's type is `Unit`). The strict checker inspects binder types /// (`check_binder`), so the selector must too, or it under-detects and drives /// such a subtree to the strict arm — a spurious `debug_typecheck` panic. -fn has_pre_desugar_artifacts(expr: &Expr) -> bool { +fn has_pre_channelize_artifacts(expr: &Expr) -> bool { fn ty_has_transient(ty: &Type) -> bool { if matches!(ty, Type::History { .. }) { return true; @@ -1359,7 +1359,7 @@ fn has_pre_desugar_artifacts(expr: &Expr) -> bool { TypedExprNode::Defer | TypedExprNode::Feed { .. } | TypedExprNode::Define { .. } ) || ty_has_transient(&expr.ty) || binder_has_transient(expr) - || expr.any_child(has_pre_desugar_artifacts) + || expr.any_child(has_pre_channelize_artifacts) } // --------------------------------------------------------------------------- @@ -1369,7 +1369,7 @@ fn has_pre_desugar_artifacts(expr: &Expr) -> bool { /// Enforce the second-class `Mut` discipline (design doc /// `src/ccl/design/mutability.md`, "No aliasing: `Mut` values are /// second-class (downward-only)"): a post-inference structural pass over the -/// fully-typed, still-`Mut`-bearing tree. Runs *after* [`check_pre_desugar`] +/// fully-typed, still-`Mut`-bearing tree. Runs *after* [`check_pre_channelize`] /// and *before* `inline`, so it sees the pre-inline `Apply`/parameter /// structure (rule 1's argument check) and the coalesced `.ty` slots. Every rule /// reads the binder's `ty`; none reads an annotation, which inference has already @@ -1691,14 +1691,14 @@ fn check_mut_write_targets_go( /// /// Pass-internal helpers (lambda elimination, substitution, simplify) call this /// after each rewrite to localize *which* operation first produced an ill-typed -/// tree. Routes through [`check_pre_desugar`], which self-selects strictness: a +/// tree. Routes through [`check_pre_channelize`], which self-selects strictness: a /// (sub)tree carrying defer artifacts (a `Feed`/`Infer` channel type — which -/// `substitute` now sees, since inline runs before desugar) is checked at the -/// relaxed `PreDesugar` level; a fully-desugared tree is checked strictly (the +/// `substitute` now sees, since inline runs before channelize) is checked at the +/// relaxed `PreChannelize` level; a fully-channelized tree is checked strictly (the /// `typecheck` bar). /// /// Gated behind the opt-in `deep-typecheck` feature. `compile_program` already -/// runs [`check_pre_desugar`] at every *pass boundary* — those walls are the +/// runs [`check_pre_channelize`] at every *pass boundary* — those walls are the /// always-on correctness net — so this per-op version only adds localization. /// It is O(subtree) and fires once per rewrite, so on nested comprehensions it /// is superlinear (O(rewrites) × O(subtree)) and dominated debug/test compile @@ -1707,7 +1707,7 @@ fn check_mut_write_targets_go( pub fn debug_typecheck(expr: &Expr) { #[cfg(feature = "deep-typecheck")] assert_eq!( - check_pre_desugar(expr), + check_pre_channelize(expr), Ok(()), "Failed post-transform typecheck: {}", crate::ccl::symbolic::symbolic_typed(expr) @@ -1765,7 +1765,7 @@ mod tests { BinOpKind::Arithmetic(ArithmeticKind::Add), Expr::lit(Lit::Int(1)), ); - // The blame is the innermost frame that saw the error — the `Var` node, + // The blame is the innermost rule that saw the error — the `Var` node, // not the enclosing `BinOp` that propagated it. let blamed = match &expr.node { crate::ccl::TypedExprNode::BinOp { left, .. } => left.node_id(), diff --git a/src/ccl/infer/check.rs b/src/ccl/infer/check.rs index f4b5a09f..77f3c06d 100644 --- a/src/ccl/infer/check.rs +++ b/src/ccl/infer/check.rs @@ -141,8 +141,8 @@ impl Typing for CheckCtx { // across the suite: it never fires. let bases: Option> = operands.iter().map(|t| offered_base(t)).collect(); let Some(bases) = bases else { - // Pre-desugar residue (a `Feed` handle, an un-eliminated `Mut`, a - // still-`Infer` position under `Strictness::PreDesugar`) is not something + // Pre-channelize residue (a `Feed` handle, an un-eliminated `Mut`, a + // still-`Infer` position under `Strictness::PreChannelize`) is not something // this rule can judge — the strictness wall decides whether a residual // type is tolerable at this point in the pipeline. return Ok(assoc.map(|_| self.fresh())); @@ -238,8 +238,10 @@ impl Typing for CheckCtx { // predicates — then return the (owned) body type unchanged rather than // cloning `bound_expr` for a no-op discharge. if crate::ccl::subst::type_free_vars(&body_ty).contains(name) { - // A discharge template is not a tree node: it is cloned again at - // every read, and that read is where the sibling is minted. + // A type-level discharge. The template keeps its ids because a + // template is not a tree node — it is cloned again at every read, and + // that read is where the sibling is minted. (Not because predicates + // are out of the id domain; they are in it.) crate::ccl::subst::Subst::discharge(name, bound_expr.clone_preserving_ids()) .apply_type(&body_ty) } else { @@ -267,12 +269,12 @@ impl Typing for CheckCtx { at: &dyn Fn() -> String, ) -> Result<(Type, Type), LocatedInferError> { // Destructure the resolved type directly (no inference vars), and — - // pre-desugar only — read through a transparent handle to the value it + // pre-channelize only — read through a transparent handle to the value it // wraps: a `Mut` history to its value (a `Mut`-typed collection used as a // for-loop source derefs to the collection), a defer's `Feed` to its // channel. Both mirror the solver's transparent-read rule that Emit applies // when it destructures the same position, so Check and Emit agree at the - // consistency wall; post-desugar/-erasure trees carry neither type. + // consistency wall; post-channelize/-erasure trees carry neither type. let mut peeled = t.peel_refinements(); while let Some(value) = peeled.mut_value_type() { peeled = value.peel_refinements(); @@ -359,7 +361,7 @@ fn check_node(expr: &mut Expr, ctx: &mut CheckCtx) -> Result Result<(), Vec> { // Check-mode failures are compiler bugs (a pass produced an ill-typed // tree), and every caller either `.expect()`s them or renders them // without source context, so the blame nodes are dropped here rather - // than plumbed through `typecheck`/`check_pre_desugar`. They are + // than plumbed through `typecheck`/`check_pre_channelize`. They are // recorded per error, so surfacing them is a signature change away when // a caller wants an underlined report. Err(ctx.errors.into_iter().map(|e| e.error).collect()) diff --git a/src/ccl/infer/context.rs b/src/ccl/infer/context.rs index a1dba15f..41c6ccb2 100644 --- a/src/ccl/infer/context.rs +++ b/src/ccl/infer/context.rs @@ -581,7 +581,7 @@ impl Typing for InferCtx { .lower_mut() .push(crate::ccl::Bound::with_subst( result, - crate::ccl::subst::Subst::discharge(&x, argument.clone()), + crate::ccl::subst::Subst::discharge(&x, argument.clone_preserving_ids()), )); Ok(applied) } diff --git a/src/ccl/infer/emit.rs b/src/ccl/infer/emit.rs index 48178752..f7bd34d7 100644 --- a/src/ccl/infer/emit.rs +++ b/src/ccl/infer/emit.rs @@ -33,7 +33,7 @@ use crate::ccl::infer::solver::traits::Trait; /// All recursion (including the generic `Typing::subexpr` impl) routes through /// here, so the mark is maintained for every emitted node. /// -/// The innermost frame wins for free: a nested rule overwrites the mark for its +/// The innermost rule wins for free: a nested rule overwrites the mark for its /// own extent, so an error is stamped with the node that raised it, not with an /// ancestor that propagated it. Nothing is read after the walk unwinds. /// @@ -48,7 +48,7 @@ use crate::ccl::infer::solver::traits::Trait; /// carry N nodes with no further change here. pub(super) fn emit_node(expr: &mut Expr, ctx: &mut InferCtx) -> Result { let prev = ctx.enter_node(expr.node_id()); - // One frame per node over the whole tree; grow on demand, as the other + // One stack frame per node over the whole tree; grow on demand, as the other // pass-level walks do. let result = stacker::maybe_grow(512 * 1024, 1024 * 1024, || emit_node_inner(expr, ctx)); ctx.leave_node(prev); @@ -60,12 +60,30 @@ pub(super) fn emit_node(expr: &mut Expr, ctx: &mut InferCtx) -> Result Result { // Compute the label before the mutable borrow so Case can pass it to emit_case. let label = symbolic(expr); + // The literal's own id, taken before the walk borrows the node — the node the + // `Lit` rule records its singleton predicate against. See there. + let node_id = expr.node_id(); // The `Lambda` rule reads the node's own type for its kind (see // `emit_lambda`), taken before the walk borrows the node. let recorded_ty = expr.ty.clone(); let has_ann = expr.user_annotation.is_some(); let mut ty = match &mut expr.node { - TypedExprNode::Lit(lit) => ctx.lit_singleton(lit), + TypedExprNode::Lit(lit) => { + // A literal's type is its singleton, `{Int | __elem == n}`, and the + // three nodes of that `__elem == n` term are minted *here* — the + // predicate is a pure function of the literal value, memoized per + // pass, so it is born the first time each distinct value is seen. + // + // The literal's own node is the slot, which is the edge + // `predicate-lineage-report` records as missing: nothing used to link + // a singleton refinement back to the literal the user wrote. + let _g = crate::ccl::lineage::enter( + node_id, + "infer.lit_singleton", + crate::ccl::lineage::Nature::Machinery, + ); + ctx.lit_singleton(lit) + } // Resolve a variable through its bound scheme. A monomorphic binder // freshens nothing and returns its type verbatim. A *polymorphic* `let` diff --git a/src/ccl/infer/mod.rs b/src/ccl/infer/mod.rs index 3b5ad352..206b20f2 100644 --- a/src/ccl/infer/mod.rs +++ b/src/ccl/infer/mod.rs @@ -535,7 +535,7 @@ pub(crate) fn run( // instances and a use-site coalesce *rebuilds* a predicate rather than // mutating one shared with the definition — occurrences share no mutable // state, so nothing needs to be kept in sync across them. - // Each coalesce error arrives blamed on the node whose frame raised it, so + // Each coalesce error arrives blamed on the node whose rule raised it, so // this pass's (potentially several) errors need no post-hoc attribution. let errors = coalesce_pass(expr); if !errors.is_empty() { diff --git a/src/ccl/infer/solve.rs b/src/ccl/infer/solve.rs index 3f3743c6..751990ed 100644 --- a/src/ccl/infer/solve.rs +++ b/src/ccl/infer/solve.rs @@ -1076,7 +1076,7 @@ fn coalesce_node(expr: &mut Expr, level: Level, ctx: &mut CoalesceCtx) { // for its own extent, so an error is blamed on the node that raised it // rather than on an ancestor. Mirrors `emit_node` / `check_node`. let prev = std::mem::replace(&mut ctx.current_node, expr.node_id()); - // One frame per node over the whole tree; grow on demand, as the other + // One stack frame per node over the whole tree; grow on demand, as the other // pass-level walks do. stacker::maybe_grow(512 * 1024, 1024 * 1024, || { coalesce_node_inner(expr, level, ctx) @@ -1470,8 +1470,10 @@ fn coalesce_node_inner(expr: &mut Expr, level: Level, ctx: &mut CoalesceCtx) { // refinement predicates) does any work; skip cloning the bound // expression when the discharge would be vacuous. if crate::ccl::subst::type_free_vars(&body.ty).contains(&binding.name) { - let sigma = - crate::ccl::subst::Subst::discharge(&binding.name, (**bound_expr).clone()); + let sigma = crate::ccl::subst::Subst::discharge( + &binding.name, + bound_expr.clone_preserving_ids(), + ); Some(sigma.apply_type(&body.ty)) } else { Some(body.ty.clone()) @@ -1498,7 +1500,7 @@ fn coalesce_node_inner(expr: &mut Expr, level: Level, ctx: &mut CoalesceCtx) { // by `mut_elim`, several passes after closure is demanded. So a refinement that // mentions the binder cannot be closed at this point, and the program is // rejected with a source position rather than left to trip the debug-only scope - // net or, in release, to reach the pre-desugar wall as a surviving mutable type. + // net or, in release, to reach the pre-channelize wall as a surviving mutable type. // Why that is staging rather than impossibility, and what lifting it would take: // see `InferError::MutableInRefinedType`. TypedExprNode::MutDecl { binding, body, .. } => { @@ -1781,19 +1783,6 @@ pub(super) fn specialize_use(use_expr: &mut Expr, frame_idx: usize, ctx: &mut Co } let base_name = frame.name.clone(); let cutoff = frame.cutoff; - // A freshened, independently-identified copy of the definition: `Clone` - // mints a new `NodeId` for every node, so N specializations cannot collide on - // one id. The clone itself covers the `walk_children` domain only: a predicate - // rides its type slot behind an `Rc` that `Type`'s `Clone` shares. - // `freshen_expr_type_slots` below re-mints those interiors separately, through - // `freshen_refinement_predicate`. - // - // TODO(mono-record): nothing captures these copies. `on_copy` records only - // into an open step, and no recorder spans inference. Whichever change adds - // one must open the step **before** this clone, because the clone is what - // fires `on_copy`: a step entered after it watches every pair fall on the - // floor and leaves the whole specialization `Unexplained`. - let mut clone = frame.def.clone(); // A monomorphization name carrying the source binding as provenance and a // globally-fresh uid for identity — so it can neither capture nor be // captured (the uid is what the old `__mono{N}` counter hand-rolled). @@ -1807,6 +1796,28 @@ pub(super) fn specialize_use(use_expr: &mut Expr, frame_idx: usize, ctx: &mut Co // `solver::freshen_expr_type_slots` / `freshen_above`), so the clone's // predicates are proper freshen instances sharing no live inference state // with the definition — and no mutable state to keep in sync with it. + // + // Sink for the clone's `on_copy` pairs, which each keep their own origin + // rather than inheriting the slot. The use site is the slot: it is the node + // this rewrite is performed for, and already what a failed pin blames. + let _spec = crate::ccl::lineage::enter( + use_expr.node_id(), + "mono.specialize", + crate::ccl::lineage::Nature::Expansion, + ); + // The clone must come *after* the recording opens. `Clone` re-mints every + // `NodeId` in the copy, so N specializations cannot collide on one id, and + // each re-mint fires `on_copy(origin, fresh)` — complete parentage on its + // own, but only an open recording captures it. Cloning first leaves every pair + // uncaptured and the whole specialization folds as `Unexplained`; measured at + // 28 such nodes on `generator_pipeline` before this ordering was fixed. + // + // This clone re-mints the `walk_children` domain only: a predicate rides its + // type slot behind an `Rc` that `Type`'s `Clone` shares. + // `freshen_expr_type_slots` below re-mints those interiors separately, + // through `freshen_refinement_predicate`, and its copies are captured by this + // same recording. + let mut clone = frame.def.clone(); let mut fresh = FreshenCache::new(); // Quantified channel-domain names must instantiate to the SAME names the // use site's pass-1 instantiation minted — a rigid name, unlike a @@ -1831,7 +1842,7 @@ pub(super) fn specialize_use(use_expr: &mut Expr, frame_idx: usize, ctx: &mut Co .and_then(|()| constrain_subtype(&use_expr.ty, &clone.ty, &mut cache)); if let Err(e) = pinned { // Blamed on the use site, which is the node whose demanded type the pin - // failed to satisfy (and the node whose frame would claim it anyway). + // failed to satisfy, and the node this specialization's recording names. ctx.errors.push(LocatedInferError { error: map_constrain_err(e, "monomorphization specialization"), node_id: use_expr.node_id(), @@ -1953,13 +1964,22 @@ pub(super) fn coalesce_generalized_let(expr: &mut Expr, level: Level, ctx: &mut // binding this rebuild deletes — see `src/ccl/design/type-inference.md`, // "Typechecking a never-called definition", for why that dangling reference is // unobservable today and what fixes it. + // + // The generalized `let` is the slot: the chain of K specialized layers + // replaces it, one origin and K products. + let _chain = crate::ccl::lineage::enter( + expr.node_id(), + "mono.coalesce_let", + crate::ccl::lineage::Nature::Expansion, + ); let mut result = body; for spec in frame.specs.into_iter().rev().filter(|s| s.referenced) { // The discharge only does work when the specialization binder is free // in the body type's refinement predicates; skip cloning `spec.def` // otherwise (it is still moved into the rebuilt `let` below). let body_ty = if crate::ccl::subst::type_free_vars(&result.ty).contains(&spec.name) { - crate::ccl::subst::Subst::discharge(&spec.name, spec.def.clone()).apply_type(&result.ty) + crate::ccl::subst::Subst::discharge(&spec.name, spec.def.clone_preserving_ids()) + .apply_type(&result.ty) } else { result.ty.clone() }; diff --git a/src/ccl/infer/solver/constrain.rs b/src/ccl/infer/solver/constrain.rs index 5d540c7e..dfd82f9b 100644 --- a/src/ccl/infer/solver/constrain.rs +++ b/src/ccl/infer/solver/constrain.rs @@ -799,7 +799,7 @@ fn constrain_go_impl( // coalesces to the bare channel, and monomorphization's two-way pin then // meets that view against the definition's feed channel. Align it with // the reconstructed channel function. Structural validation of *genuine* - // misuse (feeding a plain collection) still lands in desugar's checks. + // misuse (feeding a plain collection) still lands in channelize's checks. ( Type::Fun { .. }, Type::History { diff --git a/src/ccl/infer/solver/scheme.rs b/src/ccl/infer/solver/scheme.rs index 41fa8a68..c95e5c80 100644 --- a/src/ccl/infer/solver/scheme.rs +++ b/src/ccl/infer/solver/scheme.rs @@ -422,9 +422,10 @@ fn freshen_watches( /// /// A sharing fix here has to keep one id-set per term: reusing one rebuilt `Rc` /// across the slots that shared an `Rc` going in is one term riding many slots, -/// and returning the origin `Rc` when the freshen is vacuous is the same. -/// Producing two *distinct* terms with equal ids is what nothing may do, and -/// nothing yet checks. +/// and returning the origin `Rc` when the freshen is vacuous is the same. Only +/// producing two *distinct* terms with equal ids is forbidden — and it is +/// checked: `context.rs`'s `distinct_predicate_terms_never_share_a_node_id` +/// reports that as a `predicate-vs-predicate` collision. fn freshen_refinement_predicate( lim: Level, r: &Refinement, diff --git a/src/ccl/inline.rs b/src/ccl/inline.rs index 7e27d52f..53e5e425 100644 --- a/src/ccl/inline.rs +++ b/src/ccl/inline.rs @@ -53,7 +53,7 @@ //! //! This pass runs **before** [`crate::ccl::channelize`] (so the unified //! letrec phase can route an in-loop feed against inlined writers, and a -//! defer-mediating UDF reaches its call site before desugar routes it), so it +//! defer-mediating UDF reaches its call site before channelize routes it), so it //! *does* see [`Defer`]/[`Feed`]/[`Define`] nodes and `Type::History` (feed) domains. //! Beta-reduction goes through the defer-aware [`crate::ccl::subst::Subst`] //! engine, whose `Feed`/`Define` arms rename a fed-to handle correctly when a @@ -69,6 +69,7 @@ use crate::ccl::{ Expr, Lit, Name, Refinement, Type, TypedExprNode, ccl_utils::{PredMemo, is_free, walk_refined_predicates_mut}, lambda_elim::substitute, + lineage, }; // --------------------------------------------------------------------------- @@ -231,6 +232,10 @@ fn inline_impl(expr: Expr) -> Expr { && !is_let_bound(repl_name, &body) && !is_mut_written(repl_name, &body) { + // Alias collapse: the `Let` and its `Var` bound-expr die, the + // body is promoted. Nothing is minted, so the recording exists + // only to own the substitution's copies. + let _g = lineage::enter(node_id, "inline.alias", lineage::Nature::Machinery); return substitute(body, &binding.name, &bound_expr); } @@ -247,6 +252,7 @@ fn inline_impl(expr: Expr) -> Expr { // Let bindings (e.g. `let y = (let x = Defer in …) in …` after // expanding a defer-returning UDF) are eligible for the alias // and lift rewrites on the second pass. + let _g = lineage::enter(node_id, "inline.udf", lineage::Nature::Expansion); return inline_impl(inline_and_beta_reduce( body, &binding.name, @@ -389,6 +395,14 @@ fn inline_and_beta_reduce(expr: Expr, name: &Name, lambda: &Expr, memo: &PredMem param.ty, argument.ty ); + // Beta reduction, recorded against the `Apply` node it + // collapses. It needs no `FrameGuard::also_consumes`: the + // `Apply` and the `Lambda` both vanish, and neither has to be + // named, because the boundary difference reports both. The + // promoted `body` keeps its own id and is its own self-edge, and + // the substituted argument's copies arrive through `on_copy` as + // copies of the argument's own interior, which is what they are. + let _g = lineage::enter(node_id, "inline.beta", lineage::Nature::Expansion); return substitute(*body, ¶m.name, &argument); } // Not a Lambda (e.g. the bound expression is Var("id") rather diff --git a/src/ccl/lambda_elim.rs b/src/ccl/lambda_elim.rs index 31a229b9..94cb798b 100644 --- a/src/ccl/lambda_elim.rs +++ b/src/ccl/lambda_elim.rs @@ -1122,8 +1122,8 @@ fn elim_lambda_impl( // to the bound expression (design §6.2 move-site rule) — the same // substitution inference's let-closing and `emit_let` apply, so // the post-elim check's reconstruction reconciles structurally. - let let_ty = - crate::ccl::subst::Subst::discharge(&v, new_def.clone()).apply_type(&result_ty); + let let_ty = crate::ccl::subst::Subst::discharge(&v, new_def.clone_preserving_ids()) + .apply_type(&result_ty); Ok(Expr::let_bind(v, new_def, new_body).with_ty(let_ty)) } @@ -1737,10 +1737,23 @@ fn elim_lambdas_impl(ctx: &mut ElimContext, expr: Expr) -> Result { // TODO(preserve): a pure structural recursion rebuilds the same - // logical node, so this is arguably `Expr::preserve(node_id, node)` - // carrying the input's id rather than a mint. Minting here is at - // least *recorded* (via `Expr::new`); settling mint-vs-preserve for + // logical node, so `Expr::preserve(node_id, node)` carrying the + // input's id would serve as well as this mint. It would buy id + // correspondence across the pass: a pre-elim id would still name the + // same node afterwards. + // + // `planning/groupby` blocks it. That recognizer lifts a key function + // out of a refinement predicate into the term tree, and relies on + // `lambda_elim::run` re-minting every node so the lifted copy carries + // no id that is still live elsewhere. Preserving ids here lands + // duplicates at that lift, which + // `groupby_recognition_lifts_the_key_without_aliasing` pins as a + // property rather than as a mechanism. Settling mint-vs-preserve for // the catch-all arm wants its own change. + // + // Neither choice writes a lineage row: this file opens no recording, + // and `compile_program` runs `lambda_elim::run` outside every pass + // scope it opens. let mut expr = Expr::new(node).with_ty(ty); expr.user_annotation = user_annotation; expr.try_map_children(|child| elim_lambdas(ctx, child))?; @@ -2284,7 +2297,7 @@ mod tests { fn elim_and_typecheck(binder: &str, binder_ty: Type, body: Expr) -> String { let result = run(Expr::lambda(binder, binder_ty, body)).expect("lambda elimination"); assert_eq!( - crate::ccl::infer::check_pre_desugar(&result), + crate::ccl::infer::check_pre_channelize(&result), Ok(()), "the eliminated form must typecheck: {}", crate::ccl::symbolic::symbolic_typed(&result) diff --git a/src/ccl/lineage.rs b/src/ccl/lineage.rs index 30aaca27..42b6bac7 100644 --- a/src/ccl/lineage.rs +++ b/src/ccl/lineage.rs @@ -1,44 +1,65 @@ -//! Lineage data model: the per-pass rewrite log and the fold that collapses it -//! into a node↔node relation plus a source-span projection. +//! Lineage data model: the per-compile node table and the folds that collapse +//! it into a node↔node relation plus a source-span projection. //! //! # The model //! -//! Every IR node has a stable [`NodeId`]. As a pass rewrites the tree it appends -//! [`RewriteStep`]s to a [`LineageLog`]: a [`Op::Transform`] records "these input -//! ids vanished, these output ids appeared"; a [`Op::Copy`] records "these output -//! ids are freshened duplicates of that origin." Each step *separately* carries a -//! `blame` set (the upstream ids the outputs attribute to — **not** the same as -//! what they consumed), a `nature` bit (faithful expansion vs. pure machinery), -//! and a stable `label`. An untouched id is one no step mentions. +//! Every IR node has a stable [`NodeId`], unique within a tree — a pipeline +//! invariant asserted at every pass boundary by `assert_unique_node_ids`. Rows +//! are keyed by that identity, so everything below rests on it: two nodes +//! sharing an id would share one row and one attribution. //! -//! At an inspector pane boundary the intervening logs are folded once, in pass -//! order, by [`collapse`] into a [`LineageMap`] — a bidirectional node↔node +//! Every node a pass *produces* gets a row in the [`LineageTable`]: the ids the +//! rewrite consumed to produce it (`parents`), the ids it attributes to +//! (`blame` — **not** the same as what it consumed), and an interned +//! [`RewriteTag`] carrying the pass, the fidelity `nature`, and a stable +//! `label`. An id with no row was never rewritten. +//! +//! Rows are a byproduct of performing the rewrite, never a post-pass diff. A +//! pass names the node it is about to rewrite ([`enter`]), and the construction +//! hooks record every node minted while that guard is the innermost one open. +//! +//! For an inspector pane relation the rows its passes wrote are folded +//! once by [`collapse`] into a [`LineageMap`] — a bidirectional node↔node //! relation with an explicit self-edge for every id that survived — and, in //! parallel, into a [`SourceProjection`] that resolves each surviving node's -//! blame back to source spans. The fold composes away ids born and consumed -//! within the phase, and its two-sided leak check ([`Leak`]) guarantees no node -//! silently loses its history. +//! attribution back to source spans. The fold composes away ids born and +//! consumed within the phase, and its two-sided leak check ([`Leak`]) is what +//! says whether a node silently lost its history. +//! +//! The fold is order-free: it reads the rows as an edge set and sweeps them in +//! ascending [`NodeId`], which is a topological order of the definition graph +//! (see [`collapse`], "The algebra"). Write order is not chronology, and +//! [`collapse`] explains why nothing may depend on it. +//! +//! # Two columns, because they are two kinds of relation //! -//! # Two independent channels (load-bearing) +//! Both columns relate a node to other nodes, and differ in what the relation +//! asserts. Both reach [`LineageMap`], each labelling the edges it contributes: //! -//! `consumed` and `blame` are distinct and never mixed: +//! * **`parents`** — descends from. The ids the rewrite consumed to produce +//! this node, and the column the leak audit reads: a parent the fold +//! never heard of is a lineage that stops at an id describing nothing +//! ([`Leak::ParentUnknown`]). +//! * **`blame`** — related to, but not consumed. It may name ids that survive +//! the rewrite, so it is not an ancestry claim, and a reader asking "what was +//! this made from" reads the edge's label rather than its presence. //! -//! * **`consumed`** drives *fate* accounting — an id is consumed, carried, or -//! dropped — and the leak audit. Lineage edges resolve through the `roots` -//! state, which only `consumed`/`produced` move. -//! * **`blame`** drives *span* resolution and may name ids that survive the step. -//! It resolves through the `attr` projection, never through `roots`. +//! The labels compose weakest-link along a path, so that the inspector can +//! render blame or prune it once transitivity has run; [`EdgeLabels`] +//! states the composition. //! -//! Welding the two would (for example) resolve a channelized feed union — whose -//! step consumes the enclosing `Defer`/`Let` but blames the surviving fed-value -//! operands — to the `defer` keyword's span rather than the operands'. +//! Attribution reads both columns, unioned: a node's spans are its parents' +//! spans plus whatever distinct spans its blame adds. Blame is named at four +//! sites in the compiler, so for almost every node this is the spans of +//! whatever it was made from. Attribution reads no label — a span is a span +//! whichever column named the node it came from. //! //! # Domains, not passes //! -//! [`LineageMap`] is generic over its two id domains so the same relation serves -//! lowering's `SourceKey → NodeId` projection, a pane pair's `NodeId → NodeId`, and a -//! future `NodeId → OperatorId` edge. Passes live in the *data* (each step's -//! `via`/`label`), never in the type. +//! [`LineageMap`] is generic over its two id domains so the same relation can +//! serve a pane pair's `NodeId → NodeId` and a future `NodeId → OperatorId` +//! edge; both domains are `NodeId` today. Passes live in the *data* (each row's +//! [`RewriteTag`]), never in the type. use std::cell::RefCell; use std::collections::{HashMap, HashSet}; @@ -47,98 +68,48 @@ use std::hash::Hash; use crate::ccl::provenance::{NodeId, Pass}; use crate::chl_parser::ast::Span; -/// A stable, human-readable name for a rewrite, e.g. `"channelize.feed_union"` -/// or `"inline.fanout"`. Carried on every [`RewriteStep`] and surfaced through -/// [`RewriteTag`] for inspector tooltips. +/// A stable, human-readable name for a rewrite, e.g. `"channelize.cluster"` or +/// `"inline.beta"`. Fixed at the recording site, interned into every row's +/// [`RewriteTag`], and surfaced from there for inspector tooltips. pub type RewriteLabel = &'static str; -/// One rewrite's identity relation, in a single hop. -/// -/// Appended to a [`LineageLog`] while a pass runs. The two channels it carries — -/// the `op`'s consumed/produced/origin ids and the separate `blame` ids — are -/// resolved independently at [`collapse`] time (see the module docs). -#[derive(Clone, Debug, PartialEq, Eq)] -// Consumed when the passes adopt the recorder (next commit in the stack). -pub(crate) struct RewriteStep { - /// The identity relation this step performs. - pub op: Op, - /// Upstream ids the outputs *attribute* to — separate from `op`'s consumed - /// set (blame ⊥ consumption). May name ids that survive the step. Resolved - /// to spans at collapse through the projection being built, never through - /// the lineage `roots`. - pub blame: Vec, - /// Faithful expansion of a user construct vs. pure machinery. The one bit - /// the collapsed graph cannot recover; display policy derives from it later. - pub nature: Nature, - /// Stable rewrite label for tooling. - pub label: RewriteLabel, -} - -/// The identity relation a [`RewriteStep`] performs. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum Op { - /// `consumed` ids vanish; `produced` ids appear. Empty `produced` = discard. - /// - /// An id in `consumed ∩ produced` **survives while absorbing**: the fold - /// removes every consumed root, unions them (including the surviving id's - /// own), and assigns that union to each produced id — so a carried id keeps - /// its own lineage plus the lineage of what it absorbed. - Transform { - /// Ids that vanish. Each must be live at the step, else - /// [`Leak::ConsumedUnknown`]. - consumed: Vec, - /// Ids that appear. A live id here that was not also consumed is a lying - /// step, [`Leak::ProducedLive`]. - produced: Vec, - }, - /// `produced` ids mirror `origin`'s lineage (freshened copies). Silent on - /// the origin's own fate — the origin stays live, so a later step may still - /// consume it while the copies retain the lineage snapshotted here. - Copy { - /// The id whose lineage is mirrored. Must be live, else - /// [`Leak::CopyOfUnknown`]. - origin: NodeId, - /// The freshened copies. - produced: Vec, - }, -} - /// The fidelity of a node to its blamed source — the one fact the collapsed /// graph cannot recover. A trinary axis. /// /// **Work in progress.** This axis exists to carry display metadata to the /// inspector frontend, and neither the vocabulary nor the tagging is settled: the -/// three variants are a first cut, the rule assigning them is deliberately -/// structural for now (below), and `Expansion` has no production producer yet. -/// Treat a node's nature as a hint the frontend may present, not as a fact any -/// compiler decision should turn on — nothing in `ccl/` branches on it today, and -/// the per-site `label` is the durable datum. Retagging is cheap precisely -/// because of that: a label-keyed remap can recompute a different taxonomy -/// without touching how any pass records. +/// three variants are a first cut and the rule assigning them is deliberately +/// structural for now (below). Treat a node's nature as a hint the frontend may +/// present, not as a fact any compiler decision should turn on — nothing in +/// `ccl/` branches on it today, and the per-site `label` is the durable datum. +/// Retagging is cheap precisely because of that: a label-keyed remap can +/// recompute a different taxonomy without touching how any pass records. /// /// Public because it rides in the public [`RewriteTag`] (and thus -/// [`SourceAttribution`]); the recorder-facing [`RewriteStep`] carries it too. +/// [`SourceAttribution`]); a [`LoweringStep::Leaf`] carries one too. /// /// [`Source`](Nature::Source) is listed first because it is the base case: the /// root of a lowered source expression. The rule for who gets it is *structural* /// and stated in one place — see `LoweringContext::tag_source` in /// `src/ccl/lower/mod.rs`, and `design/provenance.md`, "The seam -/// (`src/ccl/context.rs`)". It is emitted **only by lowering** — the fold's -/// attributing arms and both wire validators carry debug guards that no *pass* -/// step ever carries it. On the wire a `Source`-nature tag null-compresses +/// (`src/ccl/context.rs`)". It is emitted **only by lowering**: [`attribute`] +/// debug-asserts that no *pass* rewrite carries it, and that is the one guard. +/// On the wire a `Source`-nature tag null-compresses /// (serializes as /// `rewritten: null` via [`is_source`](Nature::is_source)) so the wire stays /// byte-identical to the retired `rewritten: None` encoding. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Nature { /// The node is the root of a lowered source expression. Lowering-only; - /// guarded off the pass logs and off the wire. Note this is a *positional* + /// guarded off the pass rows and off the wire. Note this is a *positional* /// fact, not "images something the user wrote" — an interior image (a call's /// callee, a chained comparison's operands) carries `Machinery` with the /// `"lower.image"` label instead. Source, - /// Faithful expansion of a source construct (a comparison chain, a - /// comprehension, a lambda-elim combinator). + /// Faithful expansion of a source construct (an inlined UDF body, a + /// transaction's writer, a channelized defer cluster). Recorded by passes, + /// never by lowering, whose expansions carry `Machinery` with a per-rule + /// label. Expansion, /// Pure plumbing with no direct source counterpart. Machinery, @@ -149,9 +120,9 @@ impl Nature { /// `"machinery"`), shared by the per-node `ir` tree tag and the /// `SourceAttribution` query wire so the two encodings never diverge. /// - /// `"source"` must **never** actually reach the wire — a `Source`-nature tag - /// null-compresses at the emission sites (see [`is_source`](Self::is_source)); - /// the arm exists for the validators' guard and for completeness. + /// `"source"` never reaches the wire: the sole emission path branches on + /// [`is_source`](Self::is_source) first and writes `null` instead. The arm + /// exists for completeness. /// /// Compiled only under the `serde` feature (the `Serialize` impl below is its /// sole caller), so a default build sees it as dead. @@ -172,62 +143,187 @@ impl Nature { } } -/// A pass's ordered rewrite record. Passes append; the fold reads. -pub(crate) type LineageLog = Vec; - -/// Lowering's log entry: the same [`Op`]s a -/// [`RewriteStep`] carries, but anchored by literal source **spans** rather than -/// NodeId blame. -/// -/// The distinction from [`RewriteStep::blame`] is load-bearing and is exactly -/// why this is a *sibling* struct rather than one type generic over its -/// attribution domain: an **`anchor`** is a literal span *attached* at -/// construction (lowering knows the source token it is imaging right there), -/// whereas `blame` is a NodeId *reference resolved* later through the -/// accumulating projection (a pass names an upstream id whose spans it does not -/// itself hold). Attached-literal vs resolved-through-state are different -/// semantics, not two instances of one thing — and thread-local statics cannot -/// be generic, so a blame-domain generic would erase to the same at the -/// recorder boundary anyway. There is no NodeId-blame field: the one site that -/// would name an upstream id — a substitution, whose replacement takes the -/// replaced occurrence's attribution — carries the occurrence's identity instead -/// (`crate::ccl::subst`'s `as_expr_preserving`), so there is no id left to -/// resolve. Add one when a site demands it. +/// Lowering's log entry, in the two shapes lowering actually has. +/// +/// Its attribution channel is a literal source **span**, attached here at +/// construction because lowering knows the source token it is imaging right +/// there. A [`LineageTable`] row's `blame`, by contrast, is a NodeId +/// *reference* resolved later through the accumulating projection: a pass names +/// an upstream id whose spans it does not itself hold. Attached-literal and +/// resolved-through-state are different semantics, not two instances of one +/// thing, which is why lowering records into its own log rather than into the +/// table — and thread-local statics cannot be generic, so an +/// attribution-domain generic would erase to the same thing at the recorder +/// boundary anyway. There is deliberately no NodeId-blame channel here: +/// root-carry eliminated its only prospective user; add one when a site demands +/// it. #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct LoweringStep { - /// The identity relation this step performs. For lowering: a leaf mint is a - /// `Transform { consumed: [], produced: [id] }` (pure insertion); a copy - /// site is a `Copy { origin, produced }`. - pub op: Op, - /// The literal source spans the produced nodes image, attached here at - /// construction. Empty for a `Copy` (it mirrors its origin's folded entry). - pub anchor: Vec, - /// `Source` for a direct image, `Machinery` for manufactured plumbing - /// (`Expansion` is unused at lowering sites — the split folds into - /// `Machinery`, the per-rule label being the primary datum). - pub nature: Nature, - /// Stable rewrite label for tooling (`"lower.image"`, `"lower."`). - pub label: RewriteLabel, +pub(crate) enum LoweringStep { + /// A **leaf mint**: one node, imaged at one span. Lowering's ordinary + /// record — it mints from scratch, so a leaf has no lineage ancestor and is + /// a node plus the span it images, and nothing else. + Leaf { + /// The node this record is about. + id: NodeId, + /// The literal source span it images. + anchor: Span, + /// `Source` for a lowered expression's root, `Machinery` for an interior + /// image or manufactured plumbing (`Expansion` is unused at lowering + /// sites — the split folds into `Machinery`, the per-rule label being + /// the primary datum). + nature: Nature, + /// Stable rewrite label for tooling (`"lower.image"`, `"lower."`). + label: RewriteLabel, + }, + /// A **copy**: `produced` are freshened duplicates of `origin` and mirror + /// its folded entry verbatim. It carries no anchor, nature or label because + /// the fold reads none of them — uncurry's template interiors and the + /// compare-chain's second-use operands are exactly their origins' images, so + /// a tag here would be an unobservable value, and a wrong one would look + /// meaningful while being inert. + Copy { + /// The node whose folded entry the copies mirror. Must be recorded by an + /// earlier step, else [`Leak::ParentUnknown`]. + origin: NodeId, + /// The freshened duplicates. + produced: Vec, + }, } -/// Lowering's ordered record. Appended at leaf grain by -/// [`lowering_leaf`]/the copy frames; folded once at the lowering boundary by -/// [`collapse_lowering`] into the always-on lowering projection. +/// Lowering's ordered record. Appended at leaf grain by [`lowering_leaf`] and by +/// the copy-capturing recordings [`copy_frame`] opens; folded once at the +/// lowering boundary by [`collapse_lowering`] into the always-on lowering +/// projection. pub(crate) type LoweringLog = Vec; -/// A collapsed bidirectional relation between two id domains. +/// The lowering log, plus the set of [`NodeId`]s it has already explained. +/// +/// The set exists for one caller: [`lowering_predicate_leaf`], which sweeps a +/// finished refinement predicate and must **not** re-record a node that already +/// carries precise attribution from its own lowering. The fold is +/// last-write-wins, so a blanket sweep would silently replace a node's real span +/// and label with the coarse predicate ones — a loss no leak class can see, +/// because the node stays explained either way. +#[derive(Default)] +struct LoweringRecord { + log: LoweringLog, + recorded: HashSet, +} + +/// What an edge asserts about its two endpoints — a **set**, because one pair of +/// ids can carry both labels at once. +/// +/// Each label is one row column, closed transitively: +/// +/// * **ancestry** — the closure of `parents`: the downstream node descends from +/// the upstream one, every hop between them having consumed the node before it +/// to produce the next. Reflexive, so a surviving node is its own ancestor; +/// the column itself is irreflexive, since an in-place rewrite that keeps a +/// node's id is a *preserve* and records nothing. +/// * **blame** — the closure of `blame`: the downstream node is related to, but +/// did not consume, the upstream one. A blamed id may name a node still alive +/// elsewhere in the output tree, which is why it is not an ancestry claim. +/// +/// The closure is **weakest-link** ([`then`](Self::then)): a path is ancestry +/// only while every hop on it is an ancestry hop, and one blame hop anywhere +/// makes the whole path blame. Without that rule the label would decay into +/// "reachable somehow" over two hops — you do not descend from something you are +/// merely blamed on. Paths meeting at one endpoint pair [`union`](Self::union) +/// their labels, which is how a pair comes to carry both. +/// +/// The set is never empty: a label set exists only where an edge does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct EdgeLabels { + ancestry: bool, + blame: bool, +} + +impl EdgeLabels { + /// *Descends from*, alone — a `parents` hop, and the identity of + /// [`then`](Self::then): the zero-length path from a surviving node to + /// itself is ancestry, which is what makes a dense self-edge read as + /// ancestry rather than needing a special case. + pub const ANCESTRY: Self = EdgeLabels { + ancestry: true, + blame: false, + }; + + /// *Related to, but not consumed*, alone — a `blame` hop. + pub const BLAME: Self = EdgeLabels { + ancestry: false, + blame: true, + }; + + /// Whether the pair is in the ancestry relation. + pub fn has_ancestry(self) -> bool { + self.ancestry + } + + /// Whether the pair is in the blame relation. + pub fn has_blame(self) -> bool { + self.blame + } + + /// Extend a path by one hop: the weakest-link composition. + /// + /// Ancestry survives only if both the path so far and the hop are ancestry; + /// blame appears as soon as either is blame, because a path may take the + /// blame reading of any hop that offers one. Associative, with + /// [`ANCESTRY`](Self::ANCESTRY) as its identity, so the sweep can carry one + /// label per root and fold hops in any order. + #[must_use] + pub fn then(self, hop: Self) -> Self { + EdgeLabels { + ancestry: self.ancestry && hop.ancestry, + blame: self.blame || hop.blame, + } + } + + /// Both readings of two paths that reach the same endpoint pair. + #[must_use] + pub fn union(self, other: Self) -> Self { + EdgeLabels { + ancestry: self.ancestry || other.ancestry, + blame: self.blame || other.blame, + } + } +} + +/// One end of a labelled edge: the id at the far end, and what the edge to it +/// asserts. +/// +/// The accessors hand back these rather than bare ids because the label is the +/// content of the edge, not a detail to project away — a consumer that cannot +/// tell blame from ancestry cannot choose to render or prune it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Link { + /// The id at the far end of the edge. + pub id: T, + /// What the edge to it asserts. + pub labels: EdgeLabels, +} + +/// A collapsed bidirectional relation between two id domains, its edges labelled +/// by what they assert ([`EdgeLabels`]). /// -/// **Dense**: an id that survived a phase appears as its own self-edge, so there -/// is one uniform edge kind and no identity special case. Self-edges are -/// derivable (an id present in both snapshots is its own edge), so a later -/// sparse re-encoding behind the accessors is a pure re-encoding — consumers -/// must go through [`upstream`](Self::upstream) / [`downstream`](Self::downstream) -/// / [`edges`](Self::edges) and never touch the raw maps. +/// **One entry per `(upstream, downstream)` pair**, holding the label set: a +/// pair reached both by ancestry and by blame is one edge carrying both, +/// never two edges disagreeing about one pair. +/// +/// **Dense**: an id that survived a phase appears as its own ancestry self-edge, +/// so there is one uniform edge kind and no identity special case. Self-edges +/// are derivable (an id present in both snapshots is its own edge) and so are +/// the two directions from each other, so a later sparse re-encoding behind the +/// accessors is a pure re-encoding — consumers must go through +/// [`upstream`](Self::upstream) / [`downstream`](Self::downstream) / +/// [`edges`](Self::edges) and never touch the raw maps. The accessors expose the +/// labels, so that promise covers what an edge *asserts* as well as which pairs +/// exist. pub struct LineageMap { - /// upstream → downstream (fan-out). - down: HashMap>, - /// downstream → upstream (origins). - up: HashMap>, + /// upstream → labelled downstream (fan-out). + down: HashMap>>, + /// downstream → labelled upstream (origins). + up: HashMap>>, } impl LineageMap @@ -235,25 +331,37 @@ where U: Eq + Hash + Copy + Ord, D: Eq + Hash + Copy + Ord, { - /// The upstream origins of a downstream id (empty if the id is unknown to - /// the map). Sorted, deduplicated. - pub fn upstream(&self, d: &D) -> &[U] { + /// The upstream origins of a downstream id, each with what its edge asserts + /// (empty if the id is unknown to the map). Sorted by id, one entry per + /// origin. + pub fn upstream(&self, d: &D) -> &[Link] { self.up.get(d).map_or(&[], Vec::as_slice) } - /// The downstream fan-out of an upstream id (empty if the id is unknown to - /// the map). Sorted, deduplicated. - pub fn downstream(&self, u: &U) -> &[D] { + /// The downstream fan-out of an upstream id, each with what its edge asserts + /// (empty if the id is unknown to the map). Sorted by id, one entry per + /// target. + pub fn downstream(&self, u: &U) -> &[Link] { self.down.get(u).map_or(&[], Vec::as_slice) } - /// Every `(upstream, downstream)` edge, in deterministic order. Includes the - /// dense self-edges. - pub fn edges(&self) -> Vec<(U, D)> { - let mut out: Vec<(U, D)> = self + /// Every edge as `(upstream, labelled downstream)`, in deterministic order. + /// Includes the dense self-edges. + pub fn edges(&self) -> Vec<(U, Link)> { + let mut out: Vec<(U, Link)> = self .up .iter() - .flat_map(|(d, us)| us.iter().map(move |u| (*u, *d))) + .flat_map(|(d, us)| { + us.iter().map(move |u| { + ( + u.id, + Link { + id: *d, + labels: u.labels, + }, + ) + }) + }) .collect(); out.sort_unstable(); out @@ -283,8 +391,8 @@ pub struct SourceAttribution { // frontend format the tag itself rather than consuming a pre-flattened string. // // Null-compression: a `Source`-nature tag (a direct image) serializes its -// `rewritten` as `null`. Both validators carry a debug guard that a `"source"` -// nature never actually ships, so this compression boundary cannot rot. +// `rewritten` as `null`. This is the sole emission path, so a `"source"` nature +// cannot reach the wire. #[cfg(feature = "serde")] impl serde::Serialize for SourceAttribution { fn serialize(&self, serializer: S) -> Result { @@ -317,7 +425,12 @@ impl serde::Serialize for SourceAttribution { } /// How a rewritten node came to exist, for tooltips and display policy. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// +/// `Hash`/`Eq` are what let a [`LineageTable`] intern the whole triple as one +/// [`RuleId`]. The recording site fixes `label` and `nature`, the enclosing +/// [`PassScope`] supplies `via`, and both are settled before a row is written, +/// so the triple is one value rather than three columns. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct RewriteTag { /// The pass that performed the rewrite. pub via: Pass, @@ -349,210 +462,312 @@ impl RewriteTag { /// pane's instance is materialized by [`collapse`] at snapshot-serve time. pub type SourceProjection = HashMap; -/// A history-integrity violation surfaced by [`collapse`]. The fold's error -/// channel. +/// A history-integrity violation surfaced by a fold. The folds' error channel. /// /// `Leak::Duplicate` (one id at two tree positions) is deliberately *not* here: /// it is a tree invariant, checked pipeline-wide by `assert_unique_node_ids`, -/// not a collapse concern. +/// not a fold concern. Nor is any class shaped like a claim about a *rewrite*: +/// a row describes a node, so "two rewrites did X" has nowhere to live. The two +/// invariants that are properties of a record — one row per id, and every row +/// anchored through some channel — are asserted at +/// [`LineageTable::record`], at the site that would violate them. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Leak { - /// An output id with no lineage — a `fresh()` where a preserve was intended - /// (nothing produced or preserved this id). + /// An output id with no lineage — a `fresh()` where a preserve was intended: + /// no row the fold read produced it and the input pane does not hold it, so + /// nothing explains why it is in the output tree. Unexplained { output: NodeId }, - /// An input id that was neither consumed nor carried into the output — its - /// history vanished silently. - Dropped { input: NodeId }, - /// A [`Op::Transform`] consumed an id that was not live (an ordering or - /// attribution bug). - ConsumedUnknown { consumed: NodeId }, - /// A [`Op::Copy`] named an origin that was not live. - CopyOfUnknown { origin: NodeId }, - /// A [`Op::Transform`] produced an already-live id it did not also consume - /// (a lying step: it claims to mint what already exists). - ProducedLive { produced: NodeId }, - /// A [`Op::Transform`] with **both** empty `consumed` and empty `blame` - /// (index into the concatenated log) — a truly unanchored mint. Every step - /// must anchor its outputs through *some* channel: consumption or blame. A - /// consume-nothing transform that still names `blame` is the legal - /// *pure-insertion* shape (a node inserted over surviving material, - /// attributed via blame, with genuinely no lineage ancestor); only the - /// both-empty case cannot explain where its outputs came from. - EmptyConsumed { step: usize }, + /// An input id that is absent from the output pane — **the death report**, + /// not a defect. + /// + /// Deaths are the set difference `input_ids ∖ output_ids`, which is the + /// whole of it: nothing declares a fate, so nothing can over-claim one, and + /// this class fires for *every* node that dies across the fold. It is data the + /// inspector reads, never something a gate asserts against. The class that + /// *is* a defect on this side is [`Unexplained`](Leak::Unexplained) — an + /// output node no capture explains. + Died { input: NodeId }, + /// A row named a parent the fold has never heard of: no row it read + /// produced it and the input pane does not hold it. The node's + /// lineage stops at an id that describes nothing. + /// + /// **One class for both edge shapes**, deliberately. The sole parent of a + /// freshened copy and one consumed id of a fusion are the identical + /// condition — an edge to an id outside the fold — and telling them apart + /// would mean recording the *shape* of the rewrite, which the `parents` + /// column does not and should not carry: its cardinality already expresses + /// 1:1, 1:many and many:1, and nothing else about the shape was ever read. + ParentUnknown { parent: NodeId }, +} + +impl Leak { + /// Whether this is a **defect** — something the capture got wrong — rather + /// than the death report. + /// + /// The split is the gate: every class but [`Died`](Leak::Died) means the + /// record is inconsistent or incomplete, while `Died` is the relation's set + /// difference and fires on every ordinary death. Living on the enum rather + /// than at the one gate keeps the two readings from drifting apart — adding + /// a class forces the question here. + pub(crate) fn is_defect(&self) -> bool { + match self { + Leak::Died { .. } => false, + Leak::Unexplained { .. } | Leak::ParentUnknown { .. } => true, + } + } } -/// Resolve a `blame` set to a [`SourceAttribution`] through the `attr` -/// projection: the order-preserving, deduplicated union of each blamed id's -/// spans, tagged with `{via, nature, label}`. +/// The [`SourceAttribution`] a recorded node carries: the ordered, deduplicated +/// union of its attribution sources' spans, tagged with the row's rule. /// -/// Blame resolves through `attr` (the span channel), never through the lineage -/// `roots`. A blamed id absent from `attr` contributes no spans (it has no -/// known source), which is legal — empty blame or all-unknown blame yields -/// `spans: []`, the "known node, no source anchor" case. +/// **The sources are `parents` ∪ `blame`** — both channels, `parents` first and +/// then blame's distinct additions. Parentage takes precedence in the *order*, +/// which is what keeps the projection deterministic, but blame is never dropped: +/// a rewrite that names blame is saying "these outputs are also about that +/// node", not "attribute them there instead". A row with no parents (the pure +/// insertion) has blame as all there is; a row with no blame — the common case, +/// blame being named at four sites in the whole compiler — resolves through what +/// it was made from, which is why walking the lineage recovers a source location +/// for almost every node. /// -/// A private helper of [`collapse`], so it is dead exactly while `collapse` is. -#[allow(dead_code)] +/// The union is **unlabelled**: a span is a span whichever channel named the +/// node it came from, so this function reads the two columns as one sequence. +/// The distinction they carry is spent in the [`LineageMap`] instead, where each +/// column labels the edges it contributes ([`EdgeLabels`]) — `mut_elim`'s +/// `enter(stmt_id)` + `blame(for_id)` is the shape: the products descend from +/// the statement and are merely *related to* the loop keyword, and both spans +/// are theirs. +/// +/// A source id absent from `attr` contributes no spans (it has no known source), +/// which is legal: a node whose every source is unknown gets `spans: []`, the +/// "known node, no source anchor" case, deliberately distinct from a node absent +/// from the projection entirely. fn attribute( - blame: &[NodeId], - nature: Nature, - via: Pass, - label: RewriteLabel, + table: &LineageTable, + id: NodeId, + tag: RewriteTag, attr: &SourceProjection, ) -> SourceAttribution { - // No *pass* step may carry `Nature::Source`: `Source` means "this node is a + // No *pass* row may carry `Nature::Source`: `Source` means "this node is a // source construct's direct one-to-one translation", which only lowering can // produce — a later pass rewriting a node changes what it is. The guard sits - // on the attributing ARM, not on projection entries: an *inherited* Source - // tag on a preserved id in a later pane is legal and reaches the projection - // by clone, never through this fn. See `design/provenance.md`, "The lineage - // model (`src/ccl/lineage.rs`)". + // here, on the node being attributed by a rewrite, not on projection entries: + // an *inherited* Source tag on a preserved id in a later pane is legal and + // reaches the projection by clone, never through this fn. See + // `design/provenance.md`, "The lineage model (`src/ccl/lineage.rs`)". debug_assert!( - !nature.is_source(), - "a pass step carries Nature::Source (label {label:?}, via {via:?}) — \ - Source is emitted only by lowering" + !tag.nature.is_source(), + "a pass rewrite carries Nature::Source (label {:?}, via {:?}) — \ + Source is emitted only by lowering", + tag.label, + tag.via, ); let mut spans: Vec = Vec::new(); - for b in blame { - if let Some(a) = attr.get(b) { - for s in &a.spans { - if !spans.contains(s) { - spans.push(*s); + for src in table.parents(id).iter().chain(table.blame(id)) { + if let Some(a) = attr.get(src) { + for sp in &a.spans { + if !spans.contains(sp) { + spans.push(*sp); } } } } SourceAttribution { spans, - rewritten: RewriteTag { via, nature, label }, + rewritten: tag, + } +} + +/// A row's one-hop upstream edges, each with what that hop asserts: `parents` as +/// ancestry hops, `blame` as blame hops. +/// +/// **One entry per id.** A row naming an id in both columns is one hop carrying +/// both labels, not two hops — the pair `(p, x)` is a single edge, and emitting +/// it twice would leave the endpoint pair with two disagreeing labels for the +/// fold to pick between. Parents come first so the order is deterministic; the +/// linear scan is over a row's own columns, which hold a handful of ids. +fn row_hops(table: &LineageTable, x: NodeId) -> Vec<(NodeId, EdgeLabels)> { + let mut hops: Vec<(NodeId, EdgeLabels)> = Vec::new(); + let named = (table.parents(x).iter().map(|p| (*p, EdgeLabels::ANCESTRY))) + .chain(table.blame(x).iter().map(|b| (*b, EdgeLabels::BLAME))); + for (id, label) in named { + match hops.iter_mut().find(|(h, _)| *h == id) { + Some((_, labels)) => *labels = labels.union(label), + None => hops.push((id, label)), + } } + hops } -/// Fold the concatenated per-pass logs between two pane snapshots into the -/// pane-pair [`LineageMap`], the output pane's [`SourceProjection`], and any -/// integrity [`Leak`]s. +/// Fold the rows a pane relation's passes wrote into the pane-pair +/// [`LineageMap`], the output pane's [`SourceProjection`], and any integrity +/// [`Leak`]s. +/// +/// `passes` are the ones the relation spans, and they are what restricts a +/// whole-compile table to it: one table covers every session a compile opens, so +/// a row's `via` is the only thing that says which relation produced it. **An id +/// whose row lies outside `passes` is, to this relation, an ordinary un-produced +/// id** — an input-pane node if the input pane holds it, and unknown otherwise. +/// Without that restriction a `Mono`-produced input-pane id would resolve +/// straight past the post-channelize pane it is supposed to bottom out in. /// -/// `logs` is the intervening passes' logs in pass order; each step's `via` comes -/// from its owning log's [`Pass`] (passes live in the data, not the types). /// `input_ids` / `output_ids` are the two pane snapshots; `upstream_attr` is the /// input pane's already-resolved projection, which untouched ids inherit /// unchanged. /// -/// # The fold -/// -/// State is `roots: id → {input ids it descends from}`, seeded to the identity -/// (`roots[u] = {u}`) for every input id, and `attr`, seeded from -/// `upstream_attr`. Per step, in log order: -/// -/// * **[`Op::Transform`]**: every consumed id must be live; their root sets -/// union into `new_roots`; the consumed ids are removed; each produced id gets -/// `new_roots` and a fresh attribution. Survivor-carry works because a carried -/// id's own roots were in the union before removal. -/// * **[`Op::Copy`]**: the origin must be live and *stays* live; each produced -/// id snapshots a clone of the origin's roots and mirrors the origin's -/// attribution (re-tagged when `blame` is empty, else freshly attributed). -/// -/// At emit, each output id's roots become its `up` edges and the reverse `down` -/// edges — the bipartite product for N:M steps and self-edges for untouched ids -/// both fall out. Ids born and consumed within the phase never reach an output -/// and compose away. -/// -/// Exercised only by this module's tests until a pane boundary calls it, which -/// needs the per-pass logs the passes do not yet record. `collapse_lowering` is -/// the always-on sibling and shares the [`RootTracker`] core, so the fold logic -/// here is not untested — only this entry point is uncalled. -#[allow(dead_code)] +/// # The algebra +/// +/// A node's lineage annotation is a **map from input-pane ids to the label of +/// the path that reached them** ([`EdgeLabels`]): +/// +/// ```text +/// roots(x) = ⋃ { roots(p) ∘ hop(p → x) : p ∈ parents(x) ∪ blame(x) } +/// roots(x) = { x ↦ ancestry } if x is an input-pane id +/// ``` +/// +/// where `hop(p → x)` is ancestry for a `parents` edge, blame for a `blame` +/// edge and both for an id the row names in both columns; `∘` extends every path +/// in `roots(p)` by that hop, weakest-link ([`EdgeLabels::then`]); and `⋃` unions +/// the labels of paths that arrive at one root. The empty union `∅` is a row +/// whose parents are all unknown. +/// +/// Labelled root maps under that union form a commutative monoid — `union` on +/// [`EdgeLabels`] is a join, and `then` distributes over it — and commutativity +/// plus associativity, plus one row per id so no node has two definitions to +/// order, are exactly what make the fold insensitive to the order the rows were +/// written in. (Idempotence buys cheapness, not order-freeness.) +/// +/// Order-freeness is load-bearing, because write order is not chronology: rows +/// are written when their guard drops, so an enclosing rewrite's rows land after +/// the rows of the rewrites nested inside it. +/// +/// Ids born and consumed inside those passes are interior vertices on a path and +/// compose away; the self-edge for an untouched id falls out; the N:M bipartite +/// product of a fusion falls out of its product rows each holding the whole +/// consumed set as parents. +/// +/// # The fold is one ascending sweep +/// +/// Every edge runs from a smaller [`NodeId`] to a larger one: a row's parents and +/// its blame are alike ids the rewrite *read*, and the node itself was minted +/// afterwards from one process-global monotone counter (a produced id is +/// *captured* via [`on_mint`], never declared). A node is therefore never its own ancestor, so **ascending +/// `NodeId` order is a topological order** and one sweep suffices — no fixed +/// point, no memo, no cycle guard. [`sweep_metrics`] measures the falsifier (a +/// backward edge). +/// +/// A node reachable from nothing still keeps an entry holding `∅`, which is what +/// distinguishes "known, with empty lineage" from "the fold has never heard of +/// this id" ([`Leak::Unexplained`]). +/// +/// # The attribution channel rides along, and is not a monoid +/// +/// `attr` is resolved in the same sweep, and needs the same topological order for +/// a different reason: a row's attribution sources are ids that existed when the +/// rewrite ran. But attribution has no join — one node cannot carry two +/// `via`/`label` pairs — so it depends on there being exactly **one row per id**, +/// not on any algebra. That is the invariant [`LineageTable::record`] asserts at +/// write time, where the second writer is standing. pub(crate) fn collapse( - logs: &[(Pass, LineageLog)], + table: &LineageTable, + passes: &[Pass], input_ids: &HashSet, output_ids: &HashSet, upstream_attr: &SourceProjection, ) -> (LineageMap, SourceProjection, Vec) { - // Seeded from the input pane's identity roots; attr inherits `upstream_attr`. - let mut tracker = RootTracker::seeded(input_ids); + let mut leaks: Vec = Vec::new(); + + // The vertex set: every id the fold knows, in mint order — which is a + // topological order of the edges (see the doc comment). + let mut vertices: Vec = table + .rows_in(passes) + .chain(input_ids.iter().copied()) + .collect(); + vertices.sort_unstable(); + vertices.dedup(); + + let mut roots: HashMap> = HashMap::new(); let mut attr: SourceProjection = upstream_attr.clone(); - for (pass, log) in logs { - let via = *pass; - for step in log { - match &step.op { - Op::Transform { consumed, produced } => { - // A step must anchor its outputs through *some* channel: - // consumption or blame. Empty-consumed-with-blame is - // the legal pure-insertion shape; only both-empty is a leak. - let anchored = !step.blame.is_empty(); - tracker.transform(consumed, produced, anchored, false); - let out_attr = attribute(&step.blame, step.nature, via, step.label, &attr); - for p in produced { - attr.insert(*p, out_attr.clone()); - } - } - Op::Copy { origin, produced } => { - // A copy mirrors its origin's lineage, never its fate: the - // origin stays live and its roots are snapshotted here, so a - // later consume of the origin leaves these copies correct. - if !tracker.copy(*origin, produced) { - tracker.advance(); - continue; - } - let out_attr = if step.blame.is_empty() { - // Copy-mirror re-tag: as in `attribute`, no pass step may - // carry Source — only lowering emits it. - debug_assert!( - !step.nature.is_source(), - "a pass Copy step carries Nature::Source (label {:?}, via {via:?}) — \ - Source is emitted only by lowering", - step.label, - ); - SourceAttribution { - spans: attr - .get(origin) - .map(|a| a.spans.clone()) - .unwrap_or_default(), - rewritten: RewriteTag { - via, - nature: step.nature, - label: step.label, - }, - } - } else { - attribute(&step.blame, step.nature, via, step.label, &attr) - }; - for p in produced { - attr.insert(*p, out_attr.clone()); + for &x in &vertices { + let Some(tag) = table.rule_in(x, passes) else { + // No row among these passes: an input-pane id, reachable from itself and + // nothing else. A node descends from itself, so the self-edge is an + // ancestry edge. Its upstream attribution passes through unchanged. + roots.insert(x, HashMap::from([(x, EdgeLabels::ANCESTRY)])); + continue; + }; + + let mut r: HashMap = HashMap::new(); + for (p, hop) in row_hops(table, x) { + debug_assert!( + p < x, + "row {x:?} names the younger id {p:?} as an upstream — both columns hold \ + ids the rewrite read before it minted, so ascending NodeId order must be \ + a topological order of the definition graph", + ); + // An upstream older than `x` is already resolved if the fold knows + // it at all. No entry means it is neither an input-pane id nor + // produced here: an *ancestry* hop there is a lineage stopping at an id + // that describes nothing, while a blame-only hop is the same + // silence `attribute` keeps for a blamed id with no known spans — + // blame is a pointer at material the relation need not hold, so it + // contributes no edge and no class. + match roots.get(&p) { + Some(pr) => { + for (root, path) in pr { + let composed = path.then(hop); + r.entry(*root) + .and_modify(|l| *l = l.union(composed)) + .or_insert(composed); } } + None if hop.has_ancestry() => leaks.push(Leak::ParentUnknown { parent: p }), + None => {} } - tracker.advance(); } + roots.insert(x, r); + + let a = attribute(table, x, tag, &attr); + attr.insert(x, a); } - // Emit. Each output id's surviving roots become its up-edges (and the + // Emit. Each output id's labelled roots become its up-edges (and the // mirrored down-edges); sorted for determinism. - let mut down: HashMap> = HashMap::new(); - let mut up: HashMap> = HashMap::new(); + let mut down: HashMap>> = HashMap::new(); + let mut up: HashMap>> = HashMap::new(); for o in output_ids { - match tracker.roots_of(o) { + match roots.get(o) { Some(origins) => { - let mut origins: Vec = origins.iter().copied().collect(); + let mut origins: Vec> = origins + .iter() + .map(|(u, labels)| Link { + id: *u, + labels: *labels, + }) + .collect(); origins.sort_unstable(); - for &u in &origins { - down.entry(u).or_default().push(*o); + for link in &origins { + down.entry(link.id).or_default().push(Link { + id: *o, + labels: link.labels, + }); } up.insert(*o, origins); } - None => tracker.push_leak(Leak::Unexplained { output: *o }), + // Neither an input-pane id nor produced by any row the fold read. + None => leaks.push(Leak::Unexplained { output: *o }), } } for ds in down.values_mut() { ds.sort_unstable(); } - // A live input id that never reached the output was dropped without - // explanation (a consumed id was removed from `roots`; a carried id is in - // the output as a self-edge). + // An input id absent from the output pane **died**. Nothing declares a fate, + // so this difference is the whole death report rather than a residue of one. for u in input_ids { - if tracker.is_live(u) && !output_ids.contains(u) { - tracker.push_leak(Leak::Dropped { input: *u }); + if !output_ids.contains(u) { + leaks.push(Leak::Died { input: *u }); } } @@ -565,180 +780,123 @@ pub(crate) fn collapse( .filter_map(|o| attr.get(o).map(|a| (*o, a.clone()))) .collect(); - (LineageMap { down, up }, projection, tracker.into_leaks()) + (LineageMap { down, up }, projection, leaks) } -/// The shared roots/leak core of the two folds. Owns the -/// `roots` state (`id → {input ids it descends from}`) and the accumulating -/// [`Leak`]s, exposing the per-step fate operations both [`collapse`] and -/// [`collapse_lowering`] drive. The attribution (`attr`) side lives in each fold -/// — blame-resolved-through-state for a pane pair, literal-anchor for lowering — -/// so it deliberately stays out of the tracker. -struct RootTracker { - roots: HashMap>, - leaks: Vec, - step_index: usize, +/// What one ascending sweep of [`collapse`] costs over a given set of passes, and whether +/// the sweep's premise holds. Measurement-only. +/// +/// [`backward_edges`](Self::backward_edges) is the falsifier: ascending `NodeId` +/// order is only a topological order if every edge runs from a smaller id to a +/// larger one, so a non-zero count is exactly the number of vertices the sweep +/// would have to revisit — the fixed point it claims not to need. +#[cfg(test)] +pub(crate) struct SweepMetrics { + /// Vertices the sweep visits — and, since `roots` only ever grows, its peak + /// entry count. + pub vertices: usize, + /// Lineage edges (`upstream → node`, either label) the fold reads. + pub edges: usize, + /// Edges running from a larger `NodeId` to a smaller one — the revisit + /// count. Must be zero. + pub backward_edges: usize, } -impl RootTracker { - /// A tracker seeded with the input pane's identity roots (`roots[u] = {u}`). - fn seeded(input_ids: &HashSet) -> Self { - RootTracker { - roots: input_ids.iter().map(|&u| (u, HashSet::from([u]))).collect(), - leaks: Vec::new(), - step_index: 0, - } - } - - /// An empty tracker — lowering's degeneration: no input pane, so `roots` - /// starts empty and nearly every `Transform` is a pure insertion. - fn empty() -> Self { - RootTracker { - roots: HashMap::new(), - leaks: Vec::new(), - step_index: 0, - } - } - - /// The fate side of a [`Op::Transform`]: check the unanchored-mint leak - /// (both consumed and the caller-supplied anchor empty), union-and-remove the - /// consumed roots, then assign that union to every produced id (survivor-carry - /// works because a carried id's own roots were in the union before removal). - /// `anchored` is `true` when the step carries *some* attribution anchor — - /// blame for a pass step, a literal span for a lowering leaf. - /// - /// `reimage_ok` suppresses the [`Leak::ProducedLive`] check: a *pass* step - /// producing a live id it did not consume is a lying step, but a lowering - /// leaf may legitimately **re-image** a node (last tag wins — `lower_expr` - /// re-tags an arm's already-tagged root as the construct's direct image), so - /// the lowering fold passes `true` here. Lowering leaves always have empty - /// `consumed`, so the re-image just overwrites empty roots with empty roots. - fn transform( - &mut self, - consumed: &[NodeId], - produced: &[NodeId], - anchored: bool, - reimage_ok: bool, - ) { - if consumed.is_empty() && !anchored { - self.leaks.push(Leak::EmptyConsumed { - step: self.step_index, - }); - } - let mut new_roots: HashSet = HashSet::new(); - for c in consumed { - match self.roots.remove(c) { - Some(r) => new_roots.extend(r), - None => self.leaks.push(Leak::ConsumedUnknown { consumed: *c }), - } - } - for p in produced { - // A produced id still live here was not consumed (consumed ids were - // just removed): a pass step lies about minting it. A lowering - // re-image (`reimage_ok`) is legitimate last-tag-wins, not a lie. - if self.roots.contains_key(p) && !reimage_ok { - self.leaks.push(Leak::ProducedLive { produced: *p }); +/// Measure the cost without folding: enumerate the same vertices and edges +/// [`collapse`] sweeps and count any edge that runs backwards. +#[cfg(test)] +pub(crate) fn sweep_metrics( + table: &LineageTable, + passes: &[Pass], + input_ids: &HashSet, +) -> SweepMetrics { + let mut vertices: HashSet = table.rows_in(passes).collect(); + vertices.extend(input_ids.iter().copied()); + let (mut edges, mut backward_edges) = (0usize, 0usize); + for x in table.rows_in(passes) { + for (p, _) in row_hops(table, x) { + edges += 1; + if p >= x { + backward_edges += 1; } - self.roots.insert(*p, new_roots.clone()); - } - } - - /// The fate side of a [`Op::Copy`]: the origin must be live and *stays* live; - /// each produced id snapshots a clone of the origin's roots. Returns `false` - /// (recording [`Leak::CopyOfUnknown`]) when the origin is not live, so the - /// caller skips the attribution mirror. - fn copy(&mut self, origin: NodeId, produced: &[NodeId]) -> bool { - let Some(origin_roots) = self.roots.get(&origin).cloned() else { - self.leaks.push(Leak::CopyOfUnknown { origin }); - return false; - }; - for p in produced { - self.roots.insert(*p, origin_roots.clone()); } - true - } - - /// Advance the step counter (drives [`Leak::EmptyConsumed`]'s index). - fn advance(&mut self) { - self.step_index += 1; - } - - /// The roots of an id, if live. - fn roots_of(&self, id: &NodeId) -> Option<&HashSet> { - self.roots.get(id) - } - - /// Whether `id` is live. - fn is_live(&self, id: &NodeId) -> bool { - self.roots.contains_key(id) - } - - fn push_leak(&mut self, leak: Leak) { - self.leaks.push(leak); } - - fn into_leaks(self) -> Vec { - self.leaks + SweepMetrics { + vertices: vertices.len(), + edges, + backward_edges, } } /// Fold a [`LoweringLog`] into the always-on **lowering projection** and any -/// integrity [`Leak`]s. The lowering degeneration of -/// [`collapse`], sharing its [`RootTracker`] core with three simplifications: +/// integrity [`Leak`]s. The lowering counterpart to [`collapse`], and the one +/// fold that is **sequential**: its log genuinely is chronology (leaf entries +/// are appended at construction, not when a guard drops), and its last-tag-wins +/// re-imaging is real semantics rather than an artifact of reading a log as a +/// sequence. Four simplifications follow from lowering minting from scratch: /// -/// * **no input pane** — lowering mints from scratch, so `roots` starts empty -/// and a leaf `Transform { consumed: [], produced: [id] }` is a pure insertion -/// with empty roots (its attribution comes from the literal `anchor`, not from -/// blame resolved through state); +/// * **no input pane** — so there is no lineage to compose. Where [`collapse`] +/// carries a set of input-pane roots per id, here every such set would be +/// empty, and what is left of it is a plain **live set**: the ids some record +/// has covered; /// * **no [`LineageMap`] output** — the lowering projection ships as pane-0 -/// spans, not edges, so there is no `up`/`down` to build and no `Dropped` class +/// spans, not edges, so there is no `up`/`down` to build and no `Died` class /// (there are no input ids to drop); -/// * **no upstream attr** — a leaf's attribution is `{spans: anchor, RewriteTag}` -/// built directly here; a `Copy` mirrors its origin's already-folded entry. +/// * **no attribution state to resolve through** — a leaf's attribution is +/// `{spans: [anchor], RewriteTag}` built directly here from its literal span; a +/// copy mirrors its origin's already-folded entry; +/// * **no one-row-per-id requirement** — a re-image is a second record for one +/// id (`lower_expr` re-tags an arm's already-tagged root as the construct's +/// direct image) and the later tag deliberately wins. /// /// Runs always-on at the lowering→pipeline handoff (the leak *checks* stay -/// debug/test-gated at the boundary). `Leak::Unexplained` is an unrecorded mint -/// (an output-tree node that no leaf produced and no copy placed); a template -/// id born, copied, and never placed composes away (born-copied-discarded — live -/// but neither input nor output, so no leak); orphaned keys are structurally -/// impossible (the projection is produced by the fold, never mutated). +/// debug/test-gated at the boundary). [`Leak::Unexplained`] is an unrecorded mint +/// (an output-tree node that no leaf produced and no copy placed); +/// [`Leak::ParentUnknown`] is a copy of an origin no earlier record covered; a +/// template id born, copied, and never placed composes away (live but not an +/// output, so no leak); orphaned keys are structurally impossible (the projection +/// is produced by the fold, never mutated). pub(crate) fn collapse_lowering( log: &LoweringLog, output_ids: &HashSet, ) -> (SourceProjection, Vec) { - let mut tracker = RootTracker::empty(); + let mut live: HashSet = HashSet::new(); + let mut leaks: Vec = Vec::new(); let mut attr: SourceProjection = SourceProjection::new(); for step in log { - match &step.op { - Op::Transform { consumed, produced } => { - // The anchor is the literal attribution channel: a leaf with a - // non-empty anchor is a legal pure insertion even with empty - // `consumed`. Both-empty is the unanchored-mint leak. - let anchored = !step.anchor.is_empty(); - tracker.transform(consumed, produced, anchored, true); - // Attribution comes straight from the literal anchor spans — no - // blame resolution (lowering knows the source token here). - let out_attr = SourceAttribution { - spans: dedup_spans(&step.anchor), - rewritten: RewriteTag { - via: Pass::Lower, - nature: step.nature, - label: step.label, + match step { + LoweringStep::Leaf { + id, + anchor, + nature, + label, + } => { + live.insert(*id); + // Attribution comes straight from the literal anchor — no + // resolution through state, because lowering knows the source + // token right here. + attr.insert( + *id, + SourceAttribution { + spans: vec![*anchor], + rewritten: RewriteTag { + via: Pass::Lower, + nature: *nature, + label, + }, }, - }; - for p in produced { - attr.insert(*p, out_attr.clone()); - } + ); } - Op::Copy { origin, produced } => { + LoweringStep::Copy { origin, produced } => { // A copy mirrors its origin's already-folded entry verbatim (the // compare-chain second-use operand and the uncurry template // interiors are exactly their origins' images/plumbing). - if !tracker.copy(*origin, produced) { - tracker.advance(); + if !live.contains(origin) { + leaks.push(Leak::ParentUnknown { parent: *origin }); continue; } + live.extend(produced.iter().copied()); if let Some(origin_attr) = attr.get(origin).cloned() { for p in produced { attr.insert(*p, origin_attr.clone()); @@ -746,16 +904,15 @@ pub(crate) fn collapse_lowering( } } } - tracker.advance(); } // Every output-tree node must be explained (produced by a leaf or a copy). // An unexplained output is an unrecorded lowering mint; this leak IS the - // coverage check (there is no separate gate). There is no `Dropped` class + // coverage check (there is no separate gate). There is no `Died` class // (no input pane). for o in output_ids { - if !tracker.is_live(o) { - tracker.push_leak(Leak::Unexplained { output: *o }); + if !live.contains(o) { + leaks.push(Leak::Unexplained { output: *o }); } } @@ -764,176 +921,485 @@ pub(crate) fn collapse_lowering( .filter_map(|o| attr.get(o).map(|a| (*o, a.clone()))) .collect(); - (projection, tracker.into_leaks()) + (projection, leaks) +} + +// =========================================================================== +// The node table: the recording, keyed by the node it describes. +// +// One row per recorded *node*, which is how every consumer asks its question +// ("where did this node come from?"), so a lookup is a hash probe and the pane +// fold is one ascending sweep over the rows a pane relation's passes wrote. +// =========================================================================== + +/// An interned [`RewriteTag`] — a [`LineageTable`] row's `rule` column. +/// +/// The whole `{via, nature, label}` triple is interned as one id because the +/// three are settled together: `via` is the session's pass, and +/// `nature`/`label` are the two literals at the [`enter`] call. There are on the +/// order of fifty distinct triples in the compiler — a property of the source, +/// not of the program being compiled — so one index buys all three columns and a +/// row carries a single handle instead of three fields. +/// +/// Only meaningful against the table that minted it: the ids are dense indices +/// into that table's tag vector, not global constants. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct RuleId(u32); + +/// One recorded node's row. Both edge kinds plus the interned rule; see +/// [`LineageTable`] for why there is no span column, and +/// [`record`](LineageTable::record) for the two invariants a row must satisfy. +struct Row { + parents: Vec, + blame: Vec, + rule: RuleId, +} + +/// Per-compile lineage keyed by the node each record describes: `NodeId → { +/// parents, blame, rule }`. +/// +/// # The two edge kinds stay separate +/// +/// `parents` are the ids a rewrite consumed to produce the node — the node the +/// recording named, or a fusion's whole consumed set. `blame` are the ids the +/// rewrite is *related to* but did not consume; they may still be alive +/// elsewhere in the tree, and they say nothing about any node's fate. Both are +/// relations over the same pair of ids and both reach the fold, so what the +/// columns buy is the **label** the fold puts on the edge ([`EdgeLabels`]). +/// Merging them would answer "what was this made from" with a node that merely +/// survives beside the answer (see the module docs, "Two columns, because they +/// are two kinds of relation"). +/// +/// # There is no span column, deliberately +/// +/// A node's spans are *derived*: walk `parents` back until the walk reaches a +/// node the lowering projection covers, which is where real spans live. That +/// walk is a handful of hops and does not lengthen as programs grow, so a span +/// column would be a denormalization paid on every row ever minted, for a +/// lookup the parent edges already answer. +/// +/// # "No row" is an expected state, not an error +/// +/// The key space is `NodeId`, and a `NodeId` can be *addressed* without ever +/// having been *recorded*. So every read tolerates an unknown id — +/// [`parents`](Self::parents) and [`blame`](Self::blame) come back empty, +/// [`rule`](Self::rule) `None` — and, the sharper half of the same rule, +/// [`deaths`](Self::deaths) considers **only ids a [`record`](Self::record) +/// actually wrote**. A death is the set difference `recorded ∖ live`, so a +/// difference taken over addressed-but-unwritten ids would report nodes that +/// never existed as deaths. Row enumeration is private for exactly that reason — +/// [`deaths`](Self::deaths) and the pane fold's `rows_in` are the operations +/// that legitimately need it, and each takes the difference against something +/// (a live set, a set of passes) rather than against the key space. +/// +/// Refinement-predicate interiors **are** recorded here. They are `TypedExpr`s +/// inside a `Type::Refinement`'s predicate, carrying real `NodeId`s from the same +/// global counter and interleaved with main-tree ids, and `collect_tree_ids` +/// enumerates them — so the fold must explain them and this table must hold their +/// rows (`design/provenance.md`, "Walking the ids"). +/// +/// This was a `TODO(predicate-rows)` and is now done for two of the three +/// crossings: a term entering a predicate is recorded (lowering sweeps the +/// finished term; inference records `singleton_predicate` against its literal), +/// and a +/// predicate being rewritten is recorded. The third — planning **raising** a +/// predicate back into the main tree — is not, and lands with the planning +/// commit; those nodes are minted below the last pane, so no relation gates them +/// yet. +/// +/// It was a **population** change, not a schema change: these ids already had +/// addresses here, so no column moved. The prior measurement that justified it +/// stands as the attribution evidence — over the eleven-program corpus at the +/// `post-inference..join-planned` audit span the residue was 1184 +/// [`Leak::ParentUnknown`] edges and nothing else, every one a predicate-interior +/// id of the input tree, and admitting them took every gated class to zero. +/// +/// The backing store is a plain map on purpose: the row *semantics* above are +/// what a reader has to check, and a paged/interned column encoding would be a +/// pure re-encoding behind these accessors — the same promise +/// [`LineageMap`] makes. +#[derive(Default)] +pub(crate) struct LineageTable { + rows: HashMap, + /// Tag by [`RuleId`] index — the interning table's forward direction. + rules: Vec, + /// Tag → its already-assigned id, so equal tags share one row column. + interned: HashMap, } -/// Order-preserving deduplicated span union (a leaf anchor is normally one span, -/// but the sink-record's whole-program anchor and future multi-span leaves stay -/// well-behaved). -fn dedup_spans(spans: &[Span]) -> Vec { - let mut out: Vec = Vec::with_capacity(spans.len()); - for s in spans { - if !out.contains(s) { - out.push(*s); +// The accessors are the table's whole contract and are exercised as such by this +// module's tests; the compiler itself only ever writes rows. +#[allow(dead_code)] +impl LineageTable { + /// The [`RuleId`] for `tag` in this table, assigning one on first sight. + /// + /// Interning is separate from [`record`](Self::record) because one closing + /// guard writes many rows under one tag: the caller interns once and hands + /// the handle to each row. + pub(crate) fn intern_rule(&mut self, tag: RewriteTag) -> RuleId { + if let Some(id) = self.interned.get(&tag) { + return *id; } + // u32 is not a real limit: the distinct-tag count is a source-code + // property — one triple per label-and-nature pair the compiler writes — + // not a function of the program being compiled. + let id = RuleId( + u32::try_from(self.rules.len()).expect("more distinct rewrite tags than a u32 indexes"), + ); + self.rules.push(tag); + self.interned.insert(tag, id); + id + } + + /// Record one produced node's row. + /// + /// Three invariants, all of them properties of the write and all of them + /// checked here, at the site that would violate one, rather than at a fold + /// that only runs when someone materializes a pane: + /// + /// * **one row per id.** Attribution has no join — a node cannot carry two + /// `via`/`label` pairs — so a second writer for an id has no answer, and + /// silently overwriting would make the surviving row a lie about which + /// rewrite made the node. Ids come from one monotone counter and + /// `produced` is captured from the construction hook, so a second write + /// means a rewrite claimed to mint what already existed. + /// * **every row is anchored through some channel** — consumption or blame. + /// A row with neither cannot explain where the node came from, which is + /// the whole content of a lineage record. (A row with blame but no parents + /// is the legal *pure insertion*: a node placed over surviving material, + /// attributed through blame, with genuinely no lineage ancestor.) + /// * **a node is not its own parent.** A node's lineage is the product of + /// its parents', so an id on both sides would define itself in terms of + /// itself — the one construct that makes an edge run backwards and forces + /// [`collapse`] to a fixed point rather than a single ascending sweep. An + /// in-place rewrite that keeps its id is a **preserve**: it records + /// nothing, and that is correct, because identity here is *referent* + /// identity and the pane resolves it by shared id. + /// + /// The checks are `debug_assert!`s because this is the construction hot + /// path — every mint under an open guard lands here — and each one costs a + /// hash probe or a scan. What is gated is the checking; the row written is + /// the same row in every build. + pub(crate) fn record( + &mut self, + id: NodeId, + parents: &[NodeId], + blame: &[NodeId], + rule: RuleId, + ) { + debug_assert!( + !self.rows.contains_key(&id), + "node {id:?} already has a lineage row — a rewrite claims to have minted an id \ + that already exists, and attribution has no join to resolve the two claims with" + ); + debug_assert!( + !parents.is_empty() || !blame.is_empty(), + "node {id:?} has neither a parent nor a blamed id — every record must anchor its \ + node through some channel: consumption or blame" + ); + debug_assert!( + !parents.contains(&id), + "node {id:?} is its own parent — a rewrite cannot consume what it produces" + ); + self.rows.insert( + id, + Row { + parents: parents.to_vec(), + blame: blame.to_vec(), + rule, + }, + ); + } + + /// The ids consumed to produce `id`; empty for an unrecorded id. Predicate + /// interiors used to be the standing example of one and no longer are — they + /// are recorded (see the type's docs). + pub(crate) fn parents(&self, id: NodeId) -> &[NodeId] { + self.rows.get(&id).map_or(&[], |r| r.parents.as_slice()) + } + + /// The ids `id`'s rewrite is related to but did not consume; empty for an + /// unrecorded id, and empty for the common case of a row that mirrors its + /// parents' attribution. + pub(crate) fn blame(&self, id: NodeId) -> &[NodeId] { + self.rows.get(&id).map_or(&[], |r| r.blame.as_slice()) + } + + /// The rewrite that produced `id`, or `None` for an unrecorded id. + pub(crate) fn rule(&self, id: NodeId) -> Option { + let row = self.rows.get(&id)?; + Some(self.rules[row.rule.0 as usize]) + } + + /// The interned handle a recorded id's row holds, or `None` for an + /// unrecorded id. The identity two rows share when they name one rewrite. + pub(crate) fn rule_id(&self, id: NodeId) -> Option { + Some(self.rows.get(&id)?.rule) + } + + /// Whether `id` has a row. + pub(crate) fn contains(&self, id: NodeId) -> bool { + self.rows.contains_key(&id) + } + + /// The rewrite that produced `id` **if that rewrite is one of `passes`**, + /// else `None`. + /// + /// This is what restricts a whole-compile table to one pane relation: to that + /// relation, an id produced by a pass it does not span is an ordinary + /// un-produced id, which is exactly how the input pane's own nodes have to + /// read for the fold to bottom out there. + pub(crate) fn rule_in(&self, id: NodeId, passes: &[Pass]) -> Option { + self.rule(id).filter(|tag| passes.contains(&tag.via)) + } + + /// The ids `passes` produced, in arbitrary order. + /// + /// Private, and the same rule [`deaths`](Self::deaths) rests on: enumerating + /// rows is only ever correct against a *set of passes* or against a live set, + /// never against the key space, since a `NodeId` can be addressed without + /// ever having been recorded. This module's folds are the only callers. + fn rows_in<'a>(&'a self, passes: &'a [Pass]) -> impl Iterator + 'a { + self.rows + .iter() + .filter(|(_, row)| passes.contains(&self.rules[row.rule.0 as usize].via)) + .map(|(id, _)| *id) + } + + /// Ids the table recorded that are absent from `live` — **the death + /// report**. Deaths are a set difference and nothing declares one, so this + /// is the whole of it. + /// + /// The difference is taken over recorded rows only, never over the key + /// *space*: the key space is a global counter, so it addresses ids this + /// compile never built, and an id that was addressed but never recorded + /// describes no node that could have died. Predicate interiors were the + /// standing example — counting over the key space would have invented a death + /// per predicate node in the program — and are now recorded and live, so they + /// cancel from both sides. Sorted, so a caller's report is deterministic. + pub(crate) fn deaths(&self, live: &HashSet) -> Vec { + let mut out: Vec = self + .rows + .keys() + .copied() + .filter(|id| !live.contains(id)) + .collect(); + out.sort_unstable(); + out + } + + /// The number of recorded nodes. + pub(crate) fn len(&self) -> usize { + self.rows.len() + } + + /// The number of distinct rewrite tags interned — the rule column's + /// cardinality. + pub(crate) fn rule_count(&self) -> usize { + self.rules.len() + } + + /// The distinct passes this table holds rows for, deduplicated. + /// + /// A tag is interned only when a row is written under it, so the interning + /// table's passes are exactly the passes that recorded something. Test-only: + /// it answers "which passes rewrote this program", which is a question about + /// a compile rather than about a node, and nothing in the pipeline asks it. + #[cfg(test)] + pub(crate) fn recorded_passes(&self) -> Vec { + let mut out: Vec = Vec::new(); + for tag in &self.rules { + if !out.contains(&tag.via) { + out.push(tag.via); + } + } + out } - out } // =========================================================================== -// The recorder: an ambient thread-local step stack that turns node construction -// into RewriteSteps as a byproduct of the rewrite (never a post-pass diff). +// The recorder: an ambient thread-local stack of open recordings that turns node +// construction into lineage rows as a byproduct of the rewrite. // -// Discipline mirrors `infer_var::ACTIVE_ARENA`: install a capture buffer at a -// pass boundary, let construction hooks feed it, drain at the boundary. The -// compile path is single-threaded (the only `thread::spawn`s are runtime I/O), -// so a per-thread stack is safe. An empty stack means recording is off — the +// Discipline mirrors `infer_var::ACTIVE_ARENA`: install a capture target at a +// boundary, let construction hooks feed it, drain at the boundary. The compile +// path is single-threaded (the only `thread::spawn`s are runtime I/O), so a +// per-thread stack is safe. An empty stack means recording is off — the // overwhelmingly common case (tests, non-recorded compiles), reduced to a cheap // emptiness check on the construction hot path. // =========================================================================== thread_local! { - /// The open steps, innermost last. Empty ⇒ recording off. A construction - /// hook ([`on_mint`]/[`on_copy`]) pushes into the innermost frame only; - /// nesting is by frame depth. + /// The open recordings, innermost last. Empty ⇒ recording off. A + /// construction hook ([`on_mint`]/[`on_copy`]) pushes into the innermost + /// one only. static STEP_STACK: RefCell> = const { RefCell::new(Vec::new()) }; - /// The log the finalized steps flush into, installed per boundary by a - /// [`RecorderSession`]. `None` ⇒ no session: a step still captures into its - /// frame, but its flush is a silent no-op (uniform single code path). The - /// [`ActiveLog`] kind routes each flush to a [`RewriteStep`] (pass) or a - /// [`LoweringStep`] (lowering). - static ACTIVE_LOG: RefCell> = const { RefCell::new(None) }; -} - -/// The kind of log a [`RecorderSession`] installs — the session-kind routing. -/// `on_mint`/`on_copy` push NodeIds regardless (they are -/// blame-domain-agnostic); a frame's flush matches this to emit the right step -/// type, and the always-on lowering leaves ([`lowering_leaf`]) append here too. -/// The lowering log, plus the set of [`NodeId`]s it has already explained. -/// -/// The set exists for one caller: [`lowering_predicate_leaf`], which sweeps a -/// finished refinement predicate and must **not** re-record a node that already -/// carries precise attribution from its own lowering. The fold is -/// last-write-wins (`attr.insert(p, out_attr)`), so a blanket sweep would -/// silently replace a node's real span and label with the coarse predicate one — -/// a loss no leak class can see, because the node stays explained either way. -#[derive(Default)] -struct LoweringRecord { - log: LoweringLog, - recorded: HashSet, -} - -enum ActiveLog { - /// A pass boundary's log (inspector-only sessions). - Pass(LineageLog), - /// Lowering's log (the always-on session, all builds). - Lowering(LoweringRecord), + /// The [`LineageTable`] a closing guard writes its rows into, installed by + /// a [`TableSession`]. `None` ⇒ no table, and the write is a silent no-op, + /// so there is one code path either way. + /// + /// Installed for a **whole compile** rather than per pass, unlike + /// [`ACTIVE_PASS`]: a row is keyed by a node id, which is unique for the + /// life of the process, so one table spans every pass a compile runs and no + /// id can collide between them. + static ACTIVE_TABLE: RefCell> = const { RefCell::new(None) }; + + /// The pass a row is tagged with, installed per pass by a [`PassScope`]. + /// `None` ⇒ no pass is being recorded: a guard still captures into itself, + /// but writes nothing when it drops. + /// + /// The pass is ambient for the scope's extent because a [`RewriteTag`] needs + /// it and the recording site cannot supply it: an [`OpenStep`] carries the + /// two literals at its [`enter`] call (`label`, `nature`) and knows nothing + /// about which pass is running, while the boundary that opens the scope + /// knows exactly that. + static ACTIVE_PASS: RefCell> = const { RefCell::new(None) }; + + /// Lowering's log, installed by the always-on [`LoweringSession`]. `None` ⇒ + /// lowering is not running. Lowering records into its own sink because its + /// attribution is a literal span rather than a reference resolved later (see + /// [`LoweringStep`]). + static ACTIVE_LOWERING_LOG: RefCell> = const { RefCell::new(None) }; } -/// An in-flight step accumulating the ids born and copied within its dynamic -/// extent. Finalized (flushed to a [`RewriteStep`]) by its [`StepGuard`]'s -/// `Drop`. The `produced`/copy sides are *captured*, not declared: this is what -/// makes recording a byproduct of construction rather than a parallel -/// annotation. +/// One in-flight recording, accumulating the ids born and copied while its guard +/// is the innermost one open. Finalized when the [`FrameGuard`] drops. The +/// produced side is captured from the construction hooks rather than declared, +/// which is what makes a row a byproduct of the rewrite. struct OpenStep { label: RewriteLabel, - /// The ids this step declared it would consume. The `produced` side is + /// Extra ids the rewrite consumed, added through + /// [`FrameGuard::also_consumes`] for a fusion. The produced side is /// discovered from the construction hooks, never declared. + /// + /// Empty at open, and — `also_consumes` having no production caller — empty + /// for every rewrite in the compiler today. See [`origin`](Self::origin). consumed: Vec, + /// The id the recording site named — the node occupying the slot about to be + /// rewritten. `None` for a [`copy_frame`], which names no node. + /// + /// Every id minted in the guard's extent takes this as a **parent**: the + /// output was made from that node. The claim says nothing about whether that + /// node dies, which is what makes it safe to name a node the rewrite keeps + /// (keep the id, mint a wrapper over a child). Death is the live-set + /// difference across a pane relation, never a record-time claim. + origin: Option, blame: Vec, nature: Nature, - /// Ids minted via `Expr::new` while this was the innermost open frame. + /// Ids minted via `Expr::new` while this guard was the innermost open one. births: Vec, - /// `(origin, fresh)` pairs reported by the freshen hooks while this was the - /// innermost open frame. + /// `(origin, fresh)` pairs the freshen hooks reported while this guard was + /// the innermost open one. copies: Vec<(NodeId, NodeId)>, } impl OpenStep { - /// Finalize this frame into the active log's [`RewriteStep`]s. + /// Finalize this recording into the installed [`LineageTable`], one row per + /// node it produced. + /// + /// Two products: /// - /// The frame emits one `Transform { consumed, produced: births }`, and its - /// captured freshen pairs flush as per-origin `Copy` steps (empty blame — a - /// copy mirrors its origin, it does not re-attribute), so a deep freshen's - /// one-pair-per-node duplication lands as one `Copy` step per freshened - /// origin. A copy's origin is always *discovered* through the `on_copy` hook, - /// never declared up front: every duplication path funnels through - /// [`TypedExpr::freshen_node_id`](crate::ccl::expr::TypedExpr::freshen_node_id), - /// so capture is total and a declared-origin frame kind would be redundant. + /// * **the mints** — every id minted in the guard's extent takes the named + /// node as its parent. With a fusion ([`FrameGuard::also_consumes`]) it + /// takes the named node plus every extra id, which is the many:1 shape and + /// the only place any id is named at record time. Either way the parents + /// are lineage edges, not fate claims, so naming a node that survives + /// costs an over-broad edge and never a phantom death. + /// * **the captured freshens** — each `(origin, fresh)` pair the `on_copy` + /// hook reported rows the fresh id on the node it was duplicated from, not + /// on the named node. A copy's origin is discovered through the hook rather + /// than declared: every duplication path runs through [`copy_id`], called + /// from `TypedExpr`'s `Clone`, so capture needs no help from the site. /// - /// A `Transform` frame whose `consumed` *and* captured `births` are both - /// empty emits no `Transform` record: it was opened purely to capture the - /// freshen pairs of a deep clone (its only output is those per-origin `Copy` - /// steps), and a `Transform { consumed: [], produced: [] }` would be an - /// unanchored no-op ([`Leak::EmptyConsumed`] were blame also empty). - fn flush_into(self, log: &mut LineageLog) { + /// A guard that minted nothing, freshened nothing and fused nothing writes no + /// row. That is the **preserve** case — an in-place mutation such as `*op = + /// BinOpKind::Concat` — and it is what lets a pass open a recording on every + /// rewrite *attempt* rather than only on the ones that fire. + fn flush_into_table(self, via: Pass) { let OpenStep { label, consumed, + origin, blame, nature, births, copies, } = self; - if !(consumed.is_empty() && births.is_empty()) { - log.push(RewriteStep { - op: Op::Transform { - consumed, - produced: births, - }, - blame, - nature, - label, + let Some(origin) = origin else { + Self::assert_copy_only(label, &consumed, &births); + Self::row_per_copy(via, label, nature, &copies); + return; + }; + debug_assert!( + !births.contains(&origin), + "recording {label:?} minted the node it named, {origin:?} — the named id is \ + read before the rewrite runs, so it cannot also be a birth", + ); + if !births.is_empty() { + let mut parents = vec![origin]; + parents.extend(consumed.into_iter().filter(|c| *c != origin)); + with_table(|table| { + let rule = table.intern_rule(RewriteTag { via, nature, label }); + for &id in &births { + table.record(id, &parents, &blame, rule); + } }); } - for (origin, produced) in group_copies(&copies) { - log.push(RewriteStep { - op: Op::Copy { origin, produced }, - blame: Vec::new(), - nature, - label, - }); + Self::row_per_copy(via, label, nature, &copies); + } + + /// Row each captured freshen on the node it duplicated. A copy mirrors its + /// origin rather than re-attributing, so it carries no blame of its own. + fn row_per_copy(via: Pass, label: RewriteLabel, nature: Nature, copies: &[(NodeId, NodeId)]) { + if copies.is_empty() { + return; } + with_table(|table| { + let rule = table.intern_rule(RewriteTag { via, nature, label }); + for &(origin, fresh) in copies { + table.record(fresh, &[origin], &[], rule); + } + }); } - /// Finalize this frame into a [`LoweringLog`]. Lowering frames are opened - /// **only** to capture ambient `Copy`s (uncurry's template-interior freshens, - /// the compare-chain operand freshens); the leaf mints append directly via - /// [`lowering_leaf`], so a lowering frame carries no consumed ids and no - /// births — only the captured per-origin copies flush here, as `Copy` - /// [`LoweringStep`]s mirroring their origins' folded entries (empty anchor). + /// A [`copy_frame`] names no node, so it has nowhere to attach a consume or + /// a mint: a row needs a parent. Anything captured into one would vanish + /// from the record rather than land somewhere wrong. A pass site that trips + /// this wants [`enter`] on the node it is rewriting; a lowering site wants + /// [`lowering_leaf`] for its mints. + fn assert_copy_only(label: RewriteLabel, consumed: &[NodeId], births: &[NodeId]) { + debug_assert!( + consumed.is_empty() && births.is_empty(), + "copy frame {label:?} captured consumes {consumed:?} and mints {births:?} — a \ + recording that names no node has nothing to attach them to. A pass site wants \ + `enter` on the node being rewritten; a lowering leaf mint belongs in \ + `lowering_leaf`", + ); + } + + /// Finalize this recording into a [`LoweringLog`]. Lowering opens a guard + /// only to capture ambient copies — uncurry's template-interior freshens and + /// the compare-chain operand freshens — while its leaf mints append directly + /// via [`lowering_leaf`]. So a lowering guard carries no consumed ids and no + /// births, and only the captured per-origin copies are written here, as + /// [`LoweringStep::Copy`]s mirroring their origins' folded entries. fn flush_into_lowering(self, rec: &mut LoweringRecord) { let OpenStep { label, - nature, + consumed, births, copies, .. } = self; - debug_assert!( - births.is_empty(), - "a lowering copy-frame captured a mint ({births:?}) — leaf mints must \ - append via lowering_leaf, frames capture only copies", - ); + Self::assert_copy_only(label, &consumed, &births); for (origin, produced) in group_copies(&copies) { rec.recorded.extend(produced.iter().copied()); - rec.log.push(LoweringStep { - op: Op::Copy { origin, produced }, - anchor: Vec::new(), - nature, - label, - }); + rec.log.push(LoweringStep::Copy { origin, produced }); } } } -/// Append a single-node leaf [`LoweringStep`] (`Transform { consumed: [], -/// produced: [id] }`, `anchor: [span]`) to the active lowering log — the -/// leaf-grain recording that `tag_source`/`tag_machinery` -/// route through. A no-op when no lowering session is installed (the lower -/// submodules' unit tests, which only inspect the tree shape) or when a pass -/// session is active (defensive: lowering leaves belong only to a lowering log). /// Record one node of a **refinement predicate**, unless it is already /// explained. /// @@ -948,21 +1414,19 @@ impl OpenStep { /// sweep's coarse ones, because the fold is last-write-wins. Measured on the /// pipeline corpus: 318 nodes would be clobbered without it, and no leak class /// would report anything, since a clobbered node is still explained. +/// `the_predicate_sweep_skips_already_recorded_nodes` pins the skip. pub(crate) fn lowering_predicate_leaf(id: NodeId, span: Span, nature: Nature, label: RewriteLabel) { if id == NodeId::PLACEHOLDER { return; } - ACTIVE_LOG.with(|slot| { - if let Some(ActiveLog::Lowering(rec)) = slot.borrow_mut().as_mut() { + ACTIVE_LOWERING_LOG.with(|slot| { + if let Some(rec) = slot.borrow_mut().as_mut() { if !rec.recorded.insert(id) { return; } - rec.log.push(LoweringStep { - op: Op::Transform { - consumed: Vec::new(), - produced: vec![id], - }, - anchor: vec![span], + rec.log.push(LoweringStep::Leaf { + id, + anchor: span, nature, label, }); @@ -970,16 +1434,17 @@ pub(crate) fn lowering_predicate_leaf(id: NodeId, span: Span, nature: Nature, la }); } +/// Append a [`LoweringStep::Leaf`] to the active lowering log — the leaf-grain +/// recording that `tag_source`/`tag_machinery` route through. A no-op when no +/// [`LoweringSession`] is installed (the lower submodules' unit tests, which +/// only inspect the tree shape). pub(crate) fn lowering_leaf(id: NodeId, span: Span, nature: Nature, label: RewriteLabel) { - ACTIVE_LOG.with(|slot| { - if let Some(ActiveLog::Lowering(rec)) = slot.borrow_mut().as_mut() { + ACTIVE_LOWERING_LOG.with(|slot| { + if let Some(rec) = slot.borrow_mut().as_mut() { rec.recorded.insert(id); - rec.log.push(LoweringStep { - op: Op::Transform { - consumed: Vec::new(), - produced: vec![id], - }, - anchor: vec![span], + rec.log.push(LoweringStep::Leaf { + id, + anchor: span, nature, label, }); @@ -987,6 +1452,57 @@ pub(crate) fn lowering_leaf(id: NodeId, span: Span, nature: Nature, label: Rewri }); } +/// Run `f` against the installed [`LineageTable`], or do nothing when no +/// [`TableSession`] is installed — the one place a flush touches the sink, so +/// "recording is off" is a single silent no-op rather than a branch per row. +fn with_table(f: impl FnOnce(&mut LineageTable)) { + ACTIVE_TABLE.with(|slot| { + if let Some(table) = slot.borrow_mut().as_mut() { + f(table); + } + }); +} + +/// RAII installer for the per-compile [`LineageTable`] — the sink every closing +/// guard writes into. +/// +/// It covers a **whole compile** rather than a pass, so it outlives and nests +/// around every [`PassScope`] that compile opens. `Drop` clears the slot, so a +/// panicking compile never leaves a stale table for the next one. +pub(crate) struct TableSession { + // Not `Copy`/`Clone`; holds the installed-table invariant for its lifetime. + _private: (), +} + +impl TableSession { + /// Install a fresh, empty table as this thread's mirror target. Non-reentrant + /// (debug-asserted): a nested install would silently split one compile's rows + /// across two tables. + pub(crate) fn install() -> Self { + ACTIVE_TABLE.with(|slot| { + let mut slot = slot.borrow_mut(); + debug_assert!( + slot.is_none(), + "a TableSession is already installed on this thread", + ); + *slot = Some(LineageTable::default()); + }); + TableSession { _private: () } + } + + /// Drain and return the table, ending the session. The `Drop` that follows + /// finds the slot already empty and is a no-op. + pub(crate) fn into_table(self) -> LineageTable { + ACTIVE_TABLE.with(|slot| slot.borrow_mut().take().unwrap_or_default()) + } +} + +impl Drop for TableSession { + fn drop(&mut self) { + ACTIVE_TABLE.with(|slot| *slot.borrow_mut() = None); + } +} + /// Group captured `(origin, fresh)` pairs into per-origin produced-lists, /// preserving first-seen origin order for a deterministic log. fn group_copies(copies: &[(NodeId, NodeId)]) -> Vec<(NodeId, Vec)> { @@ -1000,95 +1516,237 @@ fn group_copies(copies: &[(NodeId, NodeId)]) -> Vec<(NodeId, Vec)> { out } -/// Open a lineage step, returning an RAII guard that finalizes it on drop. +/// Open a **lowering** copy-only recording: it consumes nothing, mints nothing, +/// and exists to capture the `(origin, fresh)` pairs a clone's freshen reports, +/// which are written as per-origin [`LoweringStep::Copy`]s (or, under a pass +/// scope, as one row per copy). /// -/// While the guard is alive the step is the innermost open frame: every id -/// minted (via `Expr::new`) or freshened (via the freshen helpers) on this -/// thread is captured into it. `blame` is the separate attribution channel -/// (blame ⊥ consumption) — the upstream ids the outputs trace to, which may -/// differ from what the step consumes. +/// This is the one recording that **names no node**, and lowering is where that +/// shape fits: uncurry's template-interior freshens and the compare-chain +/// operand freshens duplicate nodes with no slot being rewritten, so there is no +/// id to name. Every captured copy carries its own origin from the hook, which +/// is exactly why this one can afford to declare nothing at all. A *pass* that +/// duplicates uses [`enter`] instead, naming the node the duplication is +/// performed for. /// -/// `consumed` is the only thing a frame declares up front; the produced side is -/// discovered from the construction hooks. A frame that consumes nothing and -/// mints nothing — opened purely to capture a clone's freshen pairs — is a -/// [`copy_frame`], which spells that out and has no inert arguments. -pub(crate) fn step( - label: RewriteLabel, - consumed: Vec, - blame: Vec, - nature: Nature, -) -> StepGuard { +/// `nature` is fixed at [`Machinery`](Nature::Machinery) rather than taken as an +/// argument because a lowering copy's nature is never read: a +/// [`LoweringStep::Copy`] mirrors the origin's already-folded attribution +/// *verbatim*, so a nature here would be unobservable, and a wrong one (a +/// `Nature::Source` on a copy) would look meaningful while being inert. A pass +/// copy's row *does* carry a nature that reaches the attribution, which is one +/// more reason a pass duplication belongs in [`enter`]. +pub(crate) fn copy_frame(label: RewriteLabel) -> FrameGuard { STEP_STACK.with(|s| { let mut stack = s.borrow_mut(); let depth = stack.len(); stack.push(OpenStep { label, - consumed, - blame, - nature, + consumed: Vec::new(), + origin: None, + blame: Vec::new(), + nature: Nature::Machinery, births: Vec::new(), copies: Vec::new(), }); - StepGuard { depth } + FrameGuard { depth } }) } -/// Open a **lowering** copy-only frame: it consumes nothing, mints nothing, and -/// exists solely to capture the `(origin, fresh)` pairs a clone's freshen reports, -/// which flush as per-origin [`Op::Copy`] steps. -/// -/// The distinct constructor is what keeps the frame honest, and the reason it is -/// lowering-specific is the two folds' `Op::Copy` arms: -/// [`collapse_lowering`] mirrors the origin's already-folded attribution -/// *verbatim*, so a lowering copy's `nature` and `blame` are genuinely never read -/// — passing them would be passing unobservable values, and a wrong one (a -/// `Nature::Source` on a copy frame) would look meaningful while being inert. -/// [`collapse`] does **not** mirror: a pass `Op::Copy` with empty blame builds -/// `RewriteTag { nature: step.nature, .. }`, so a pass copy-frame's nature reaches -/// the attribution and must be chosen deliberately. Such a frame opens with -/// [`step`] and names its nature. -pub(crate) fn copy_frame(label: RewriteLabel) -> StepGuard { - step(label, Vec::new(), Vec::new(), Nature::Machinery) +// =========================================================================== +// Capture keyed on node identity. +// +// A rewriting site names the node it is *about to rewrite* and declares nothing +// else. Every id minted while that guard is the innermost one open records the +// named node as its parent, which says nothing about whether the named node +// survived: death is the pane relation's live-set difference, so no site predicts +// a fate. +// +// The produced side is never declared. It is a byproduct of construction, +// discovered through the mint and copy hooks. The one recording that names no +// node is `copy_frame`, lowering's copy sink, whose captured copies each carry +// their own origin. +// =========================================================================== + +/// Open a recording over the node currently in the slot being rewritten, +/// returning an RAII guard that finalizes it on drop. +/// +/// `slot_id` is read off the node *before* the rewrite runs; every id minted +/// while this guard is the innermost one open records `slot_id` as its parent. +/// The site declares nothing further — see [`OpenStep::flush_into_table`]. +/// +/// A guard rather than a closure, for two reasons that outlive the ergonomics: +/// +/// * **The region is a scope, not an expression.** A site may talk to the open +/// recording after opening it — `mut_elim` calls [`FrameGuard::blame`] on the +/// next line, naming the `For` so the products resolve to the loop keyword's +/// span rather than the enclosing statement's. A closure taking only the +/// rewrite has no channel for that, so each extra channel would become a +/// parameter. +/// * **A channel may fire from a runtime-decided point inside the region.** +/// Whether a rewrite takes the arm that widens its attribution is decided by a +/// `match` on the node, potentially far below the `enter`. As a closure, that +/// means making a function's whole tail a closure body so one arm can reach +/// the recording. +/// +/// The guard costs nothing in expressiveness: the id is needed only at *entry*, +/// and nothing inspects the slot at exit. +/// +/// Two consequences a site does not restate: +/// +/// * **A recording is where the hooks write.** An installed table captures +/// nothing on its own; the mint and copy hooks need an open recording to +/// attach to, so a rewrite that clones or mints outside one drops its pairs on +/// the floor. That is the failure mode, not a wrong parent. +/// * **Open it after any recursion into children.** A recording adopts +/// everything minted under it, so opening one around a recursive call attaches +/// the callee's own products to this node instead of theirs. Open it around the +/// rewrite alone, and after the early returns that abandon it. +pub(crate) fn enter(slot_id: NodeId, label: RewriteLabel, nature: Nature) -> FrameGuard { + STEP_STACK.with(|s| { + let mut stack = s.borrow_mut(); + let depth = stack.len(); + stack.push(OpenStep { + label, + consumed: Vec::new(), + origin: Some(slot_id), + blame: Vec::new(), + nature, + births: Vec::new(), + copies: Vec::new(), + }); + FrameGuard { depth } + }) } -/// RAII finalizer for an open [`step`]. Popping and flushing on `Drop` is -/// panic-safe: an unwind through an open step still pops its frame (so the stack -/// is never left corrupt) and flushes what was captured before the panic. -pub(crate) struct StepGuard { - /// The stack index this frame occupied when opened, for the LIFO tripwire. +/// RAII finalizer for an open recording. Popping and writing on `Drop` is +/// panic-safe: an unwind through an open recording still pops it, so the stack +/// is never left corrupt, and writes what was captured before the panic. +/// +/// The guard is also the *only* handle on the recording it opened: the extra +/// channels ([`FrameGuard::also_consumes`], [`FrameGuard::blame`]) are inherent +/// methods on it, so a site cannot address a recording it does not hold — the +/// innermost one may belong to a callee or to an enclosing recursion. +#[must_use = "a dropped FrameGuard records nothing — bind it (`let _g = …`), \ + and note `let _ = …` drops it immediately"] +pub(crate) struct FrameGuard { + /// The stack index this recording occupied when opened, for the LIFO + /// tripwire. depth: usize, } -impl Drop for StepGuard { +impl FrameGuard { + /// Fusion (many:1): this rewrite **also** consumed `id`, a node the + /// construction hooks cannot attribute because it is not the one the site + /// named. + /// + /// The only channel that adds a *consumed* id beyond the named one, and so + /// **the only place any id is named at record time** — everything else about + /// a recording is observed. The named id joins the site's own node in the + /// products' `parents`, asserting ancestry and nothing about `id`'s fate. + /// + /// A [`copy_frame`] names no node for it to sit beside, so this is + /// meaningless there and [`assert_copy_only`] catches it. + /// + /// [`assert_copy_only`]: OpenStep::assert_copy_only + // No production caller: every rewrite in the compiler is 1:many, so each + // recording writes one parent per product. The channel is retained because + // nothing else can express the many:1 shape — a fusion onto an older + // survivor has to remint (`consumed: [S, D…] → produced: [S′]`) to keep + // parents ahead of children. Exercised by + // `a_fusion_gives_every_product_the_bipartite_product`. + #[allow(dead_code)] + pub(crate) fn also_consumes(&self, id: NodeId) { + if id == NodeId::PLACEHOLDER { + return; + } + self.with_own_frame(|top| top.consumed.push(id)); + } + + /// Additional source attribution: nodes this rewrite's products are *about* + /// beyond the one being rewritten. + /// + /// With no blame the products take the named node's spans, which is right for + /// most rewrites. Naming blame **adds** to that — attribution is the union of + /// the parents' spans and these — so a site widens the attribution rather + /// than redirecting it. + /// + /// Blame relates without claiming ancestry: these ids may name nodes that + /// survive the rewrite, so they ride the `blame` column rather than `parents` + /// and reach the pane relation as *blame* edges ([`EdgeLabels`]), which the + /// inspector can render or prune. Weakest-link closure keeps that distinction + /// alive at a distance: anything reached through one of these hops is + /// related, never descended. Naming an id here therefore asserts nothing + /// about its fate, which is what lets a site blame a node it leaves in the + /// tree. + pub(crate) fn blame(&self, ids: &[NodeId]) { + self.with_own_frame(|top| top.blame.extend(ids.iter().copied())); + } + + /// Address *this* guard's own recording, asserting it is the innermost open + /// one. + /// + /// The channels above are only meaningful about the recording the caller + /// holds; reaching whatever happens to be on top would silently retarget a + /// callee's or an enclosing recursion's. Same `debug_assert_eq!` convention + /// as the LIFO tripwire in [`Drop`](FrameGuard::drop). + fn with_own_frame(&self, f: impl FnOnce(&mut OpenStep)) { + STEP_STACK.with(|s| { + let mut stack = s.borrow_mut(); + debug_assert_eq!( + stack.len(), + self.depth + 1, + "FrameGuard channel used while it is not the innermost open recording \ + (expected depth {}, stack has {})", + self.depth, + stack.len(), + ); + if let Some(top) = stack.last_mut() { + f(top); + } + }); + } +} + +impl Drop for FrameGuard { fn drop(&mut self) { - // Pop our frame. Guards drop in LIFO order in normal control flow and on - // unwind alike; the tripwire catches a manually-mis-ordered drop. + // Pop this guard's recording. Guards drop in LIFO order in normal + // control flow and on unwind alike; the tripwire catches a + // manually-mis-ordered drop. let frame = STEP_STACK.with(|s| { let mut stack = s.borrow_mut(); debug_assert_eq!( stack.len(), self.depth + 1, - "StepGuard dropped out of LIFO order (expected depth {}, stack has {})", + "FrameGuard dropped out of LIFO order (expected depth {}, stack has {})", self.depth, stack.len(), ); stack.pop() }); let Some(frame) = frame else { return }; - // Flush to the active log if a session is installed; a no-op otherwise. - // The log kind routes the flush to the matching step type. - ACTIVE_LOG.with(|slot| match slot.borrow_mut().as_mut() { - Some(ActiveLog::Pass(log)) => frame.flush_into(log), - Some(ActiveLog::Lowering(rec)) => frame.flush_into_lowering(rec), - None => {} - }); + // Lowering records into its own log; under a pass scope the rows go to + // the table, tagged with the ambient pass. Under neither, the write is a + // silent no-op: the recording captured, and has nowhere to land. + if ACTIVE_LOWERING_LOG.with(|slot| slot.borrow().is_some()) { + ACTIVE_LOWERING_LOG.with(|slot| { + if let Some(rec) = slot.borrow_mut().as_mut() { + frame.flush_into_lowering(rec); + } + }); + return; + } + if let Some(via) = ACTIVE_PASS.with(|p| *p.borrow()) { + frame.flush_into_table(via); + } } } /// A hook called from `Expr::new` for every minted [`NodeId`]. Pushes the id -/// into the innermost open step's births, or does nothing when no step is open +/// into the innermost open recording's births, or does nothing when none is open /// (the common case — a borrow and an emptiness check). The [`PLACEHOLDER`] -/// sentinel is ignored so `Default`/`mem::take` throwaways never pollute a step. +/// sentinel is ignored, so `Default`/`mem::take` throwaways are never attributed +/// to a rewrite. /// /// [`PLACEHOLDER`]: NodeId::PLACEHOLDER pub(crate) fn on_mint(id: NodeId) { @@ -1124,7 +1782,11 @@ impl Drop for PreservingIds { /// Open a scope in which [`TypedExpr`](crate::ccl::expr::TypedExpr)'s `Clone` /// **preserves** ids instead of freshening them. /// -/// Reach for this through +/// Freshening is the default and the norm: a clone is a sibling of what it +/// copied, with its own identity and a row recording the pair. This scope +/// suppresses that, so every use is an exception that has to argue for itself. +/// +/// Reach for it through /// [`TypedExpr::clone_preserving_ids`](crate::ccl::expr::TypedExpr::clone_preserving_ids), /// never directly: the scope must cover the clone and nothing else, and a /// genuine duplication performed inside one would silently produce a @@ -1136,31 +1798,30 @@ pub(crate) fn preserve_ids() -> PreservingIds { } /// Run `f` with id-preserving clones — a scope over a whole *rewrite region*, -/// not over a copy. -/// -/// # TODO(predicate-domain): this is slated for removal. Do not add callers. -/// -/// **There is exactly one legitimate user: [`PredMemo`]'s predicate rebuild** -/// (two call sites). Everything else that needs a preserving copy has one — -/// [`TypedExpr::clone_preserving_ids`] — and should use it. A new caller here is -/// almost certainly reaching for the wrong tool: this scope silences the -/// freshening for *every* clone on the thread until `f` returns, including -/// genuine duplications a callee performs, so it can manufacture duplicate ids -/// in a way the per-copy method cannot. -/// -/// It exists because what must not mint inside a predicate rebuild is not a copy -/// but an arbitrary caller-supplied rewrite: `f` does not clone the predicate, it -/// mints *into* it (a substitution materializing a template, a rule building a -/// conjunction). Nothing records a predicate rewrite, so a node minted there is -/// one no record explains, and a `Copy` rowed against it folds as -/// [`Leak::CopyOfUnknown`]. `clone_preserving_ids` covers one copy; -/// only a scope covers a region. -/// -/// Preserving is honest here because the rebuilt term *replaces* the original -/// everywhere the walk reaches — which holds only because `uniquify` walks the -/// whole tree. It is a **scope cut**, not a design: the predicate domain needs -/// recording, and this function should go when that lands. See the vault's -/// `predicate-lineage-report` and `design/provenance.md`, "Walking the ids". +/// not over a single copy. +/// +/// # TODO(predicate-domain): do not add callers without reading this. +/// +/// **One legitimate user: [`PredMemo`] in *replacing* mode**, which is to say +/// [`uniquify`](crate::ccl::uniquify) and nothing else. Anything that needs a +/// preserving *copy* has one — [`TypedExpr::clone_preserving_ids`] — and should +/// use it. This scope silences the freshening for **every** clone on the thread +/// until `f` returns, including genuine duplications a callee performs, so unlike +/// the per-copy method it can manufacture duplicate ids. +/// +/// It exists because what must keep its identity is not a copy but an arbitrary +/// caller-supplied *rewrite*: `f` mints and copies *into* the term (a +/// substitution materializing a template, a rule building a conjunction), and +/// those products are part of the same replacement. +/// +/// The justification is **replacement, not domain**. Predicate interiors are in +/// the id domain and the fold explains them (`design/provenance.md`, "Walking +/// the ids"). What makes preserving honest here is that the rebuilt term stands in +/// for the original *everywhere* — which is true only because `uniquify` walks +/// the whole tree, and which `uniquify` asserts on every compile as a 1:1 +/// correspondence over distinct predicate terms. A rebuild whose walk misses an +/// occurrence leaves the original alive beside its replacement, and then this +/// scope puts one id-set on two live terms. /// /// [`PredMemo`]: crate::ccl::ccl_utils::PredMemo /// [`TypedExpr::clone_preserving_ids`]: crate::ccl::expr::TypedExpr::clone_preserving_ids @@ -1184,10 +1845,10 @@ pub(crate) fn copy_id(origin: NodeId) -> NodeId { } /// A hook called from the freshen helpers for every `(origin, fresh)` -/// duplication. Pushes the pair into the innermost open step's copies, or does -/// nothing when no step is open. Guards the [`PLACEHOLDER`] sentinel on both +/// duplication. Pushes the pair into the innermost open recording's copies, or +/// does nothing when none is open. Guards the [`PLACEHOLDER`] sentinel on both /// sides, as [`on_mint`] does: a placeholder origin would fold as -/// [`Leak::CopyOfUnknown`] against an id that is never live by construction. +/// [`Leak::ParentUnknown`] against an id nothing ever records. /// /// [`PLACEHOLDER`]: NodeId::PLACEHOLDER pub(crate) fn on_copy(origin: NodeId, fresh: NodeId) { @@ -1201,89 +1862,104 @@ pub(crate) fn on_copy(origin: NodeId, fresh: NodeId) { }); } -/// RAII installer for the active recording log. [`new`](Self::new) installs a -/// **pass** log ([`ActiveLog::Pass`]); [`lowering`](Self::lowering) installs a -/// **lowering** log ([`ActiveLog::Lowering`]). The matching drain -/// ([`into_log`](Self::into_log) / [`into_lowering_log`](Self::into_lowering_log)) +/// RAII installer for **lowering's** log. +/// +/// Installed unconditionally for the whole of lowering in every build, because +/// the projection it folds into is release-critical: an `InferError` resolves +/// its blame node to a span through it. [`into_log`](Self::into_log) drains and /// ends the session; `Drop` clears the slot so a panic never leaves a stale log -/// installed for the next boundary. At most one session per thread, and — since -/// the lowering session installs unconditionally in every build — it must fully -/// drain before the first pass session opens. -pub(crate) struct RecorderSession { +/// installed. At most one per thread, and it must fully drain before the first +/// [`PassScope`] opens — a guard closing while both were installed would record +/// lowering-shaped copies for a pass rewrite. +pub(crate) struct LoweringSession { // Not `Copy`/`Clone`; holds the installed-log invariant for its lifetime. _private: (), } -impl RecorderSession { - /// Install a fresh, empty **pass** log as the active recording target for - /// this thread. Non-reentrant: at most one session per thread - /// (debug-asserted). - /// - /// The pass-log counterpart to [`lowering`](Self::lowering), which is - /// always-on. Uncalled outside this module's tests until a pass boundary - /// installs a session of its own. - #[allow(dead_code)] - pub(crate) fn new() -> Self { - Self::install(ActiveLog::Pass(Vec::new())) - } - - /// Install a fresh, empty **lowering** log. The always-on session: - /// installed for the whole of lowering in every build. Its leaf entries - /// ([`lowering_leaf`]) and copy-frame flushes route to a [`LoweringLog`]. - pub(crate) fn lowering() -> Self { - Self::install(ActiveLog::Lowering(LoweringRecord::default())) - } - - fn install(log: ActiveLog) -> Self { - ACTIVE_LOG.with(|slot| { +impl LoweringSession { + /// Install a fresh, empty lowering log as this thread's recording target. + /// Non-reentrant (debug-asserted). + pub(crate) fn install() -> Self { + ACTIVE_LOWERING_LOG.with(|slot| { let mut slot = slot.borrow_mut(); debug_assert!( slot.is_none(), - "a RecorderSession is already installed on this thread", + "a LoweringSession is already installed on this thread", ); - *slot = Some(log); + *slot = Some(LoweringRecord::default()); }); - RecorderSession { _private: () } + LoweringSession { _private: () } } - /// Drain and return the recorded **pass** log, ending the session. The - /// subsequent `Drop` is then a no-op (the slot is already empty). - /// - /// Paired with [`new`](Self::new), so it is dead exactly while that is. - #[allow(dead_code)] - pub(crate) fn into_log(self) -> LineageLog { - ACTIVE_LOG.with(|slot| match slot.borrow_mut().take() { - Some(ActiveLog::Pass(log)) => log, - other => { - debug_assert!(other.is_none(), "into_log on a non-pass session"); - Vec::new() - } - }) + /// Drain and return the recorded log, ending the session. The `Drop` that + /// follows finds the slot already empty and is a no-op. + pub(crate) fn into_log(self) -> LoweringLog { + ACTIVE_LOWERING_LOG.with(|slot| slot.borrow_mut().take().unwrap_or_default().log) } +} - /// Drain and return the recorded **lowering** log, ending the session. - pub(crate) fn into_lowering_log(self) -> LoweringLog { - ACTIVE_LOG.with(|slot| match slot.borrow_mut().take() { - Some(ActiveLog::Lowering(rec)) => rec.log, - other => { - debug_assert!( - other.is_none(), - "into_lowering_log on a non-lowering session" - ); - Vec::new() - } - }) +impl Drop for LoweringSession { + fn drop(&mut self) { + ACTIVE_LOWERING_LOG.with(|slot| *slot.borrow_mut() = None); + } +} + +/// RAII installer for the ambient [`Pass`] every row is tagged with. +/// +/// One scope per pass, opened at the boundary that runs it. The pass is carried +/// here rather than named per recording because a recording site knows its +/// `label` and `nature` but not which pass is running, while the boundary that +/// opens the scope knows exactly that: one pass runs inside one scope, so the +/// pass is ambient over the scope's whole extent. (A scope spanning several +/// passes, as an audit span opens, tags every row with the one pass it names — +/// no single pass being the truthful answer there.) +/// +/// Opening a scope is what turns pass recording **on**: outside one a guard still +/// captures, but has no tag to complete a row with and writes nothing. +pub(crate) struct PassScope { + // Not `Copy`/`Clone`; holds the installed-pass invariant for its lifetime. + _private: (), +} + +impl PassScope { + /// Install `pass` as this thread's ambient recording pass. Non-reentrant + /// (debug-asserted): a nested scope would silently retag the inner pass's + /// rows on exit. + pub(crate) fn enter(pass: Pass) -> Self { + debug_assert!( + ACTIVE_LOWERING_LOG.with(|slot| slot.borrow().is_none()), + "opening a PassScope ({pass:?}) while lowering's log is still installed — a \ + closing guard would write lowering-shaped copies for a pass rewrite. Drain \ + the LoweringSession first", + ); + ACTIVE_PASS.with(|slot| { + let mut slot = slot.borrow_mut(); + debug_assert!( + slot.is_none(), + "a PassScope is already open on this thread (opening {pass:?})", + ); + *slot = Some(pass); + }); + PassScope { _private: () } } } -impl Drop for RecorderSession { +impl Drop for PassScope { fn drop(&mut self) { - // Clear on unwind (and after a plain `into_log`, harmlessly): the next - // pass must never see a stale log. - ACTIVE_LOG.with(|slot| *slot.borrow_mut() = None); + ACTIVE_PASS.with(|slot| *slot.borrow_mut() = None); } } +/// Read the installed [`LineageTable`] mid-compile, or `None` when no +/// [`TableSession`] is installed. +/// +/// The compile's table is drained at the end of the compile, so a measurement +/// that folds a span *inside* one — the lineage audit — has no other way to +/// reach the rows it just caused to be written. +pub(crate) fn with_active_table(f: impl FnOnce(&LineageTable) -> R) -> Option { + ACTIVE_TABLE.with(|slot| slot.borrow().as_ref().map(f)) +} + /// The current open-step depth on this thread — a probe for the panic-safety /// and no-op tests, which assert the stack is left clean. #[cfg(test)] @@ -1299,6 +1975,14 @@ mod tests { Span::new(start, end) } + /// The pass the tests record under. The recorder is pass-agnostic — the tag + /// only rides through to a row — so naming one constant keeps the choice + /// from looking meaningful at each call site. + const TEST_PASS: Pass = Pass::Inline; + + /// The pass set every fold test uses: the one pass its rows carry. + const PASSES: &[Pass] = &[TEST_PASS]; + fn ids() -> [NodeId; N] { std::array::from_fn(|_| NodeId::fresh()) } @@ -1307,103 +1991,76 @@ mod tests { items.into_iter().collect() } - /// The predicate sweep must not overwrite attribution a node already has. - /// - /// This is the one property of `lowering_predicate_leaf` that **no leak class - /// can see**: the fold is last-write-wins, so a node re-recorded by the sweep - /// is still perfectly *explained* — it has just silently swapped its real - /// span and label for the sweep's coarse ones. Measured on the pipeline - /// corpus, a blanket sweep clobbers 318 nodes and every gate stays green. - #[test] - fn the_predicate_sweep_skips_already_recorded_nodes() { - let [recorded, fresh] = ids::<2>(); - let session = RecorderSession::lowering(); - // A node lowered in the main tree: precise span, precise label. - lowering_leaf(recorded, span(10, 20), Nature::Source, "lower.precise"); - // The sweep runs over a predicate containing both that node and one - // minted while assembling the predicate. - lowering_predicate_leaf(recorded, span(0, 99), Nature::Machinery, "lower.sweep"); - lowering_predicate_leaf(fresh, span(0, 99), Nature::Machinery, "lower.sweep"); - let log = session.into_lowering_log(); + /// The gated subset of a leak vector — everything but the death report. The + /// hand-built tables below mostly report no deaths at all and assert on the + /// whole vector; a test that records a *real* rewrite gets `Died` for the + /// node it replaced and wants this instead. + fn defects(leaks: &[Leak]) -> Vec<&Leak> { + leaks.iter().filter(|l| l.is_defect()).collect() + } - assert_eq!(log.len(), 2, "the already-recorded node is not re-recorded"); - let for_recorded: Vec<_> = log - .iter() - .filter( - |s| matches!(&s.op, Op::Transform { produced, .. } if produced == &vec![recorded]), - ) - .collect(); - assert_eq!( - for_recorded.len(), - 1, - "exactly one entry for the lowered node" - ); - assert_eq!( - for_recorded[0].label, "lower.precise", - "its own label survives" - ); - assert_eq!( - for_recorded[0].anchor, - vec![span(10, 20)], - "its own span survives" - ); + fn sorted(mut v: Vec) -> Vec { + v.sort_unstable(); + v + } - let for_fresh: Vec<_> = log - .iter() - .filter(|s| matches!(&s.op, Op::Transform { produced, .. } if produced == &vec![fresh])) - .collect(); - assert_eq!( - for_fresh.len(), - 1, - "the assembly node is explained by the sweep" - ); - assert_eq!(for_fresh[0].label, "lower.sweep"); + /// The ids one side of a node's edges names, labels dropped — for the tests + /// whose subject is *which pairs* the fold derives rather than what they + /// assert. The label tests below read [`Link::labels`] instead. + fn ids_of(links: &[Link]) -> Vec { + links.iter().map(|l| l.id).collect() } - /// A second sweep over the same predicate adds nothing — the skip is keyed on - /// the id, not on which sweep recorded it, so overlapping predicates (one - /// term riding several type slots) cannot double-record. - #[test] - fn the_predicate_sweep_is_idempotent() { - let [n] = ids::<1>(); - let session = RecorderSession::lowering(); - lowering_predicate_leaf(n, span(1, 2), Nature::Machinery, "lower.sweep"); - lowering_predicate_leaf(n, span(3, 4), Nature::Machinery, "lower.sweep"); - let log = session.into_lowering_log(); - assert_eq!( - log.len(), - 1, - "one entry per node however many sweeps reach it" - ); - assert_eq!(log[0].anchor, vec![span(1, 2)], "the first sweep wins"); + /// What the edge to `id` asserts, or `None` when the pair is not an edge at + /// all — the two answers a labelled relation can give, kept apart. + fn labels_of(links: &[Link], id: NodeId) -> Option { + links.iter().find(|l| l.id == id).map(|l| l.labels) } - fn transform(consumed: Vec, produced: Vec, blame: Vec) -> RewriteStep { - RewriteStep { - op: Op::Transform { consumed, produced }, + /// Write one row by hand, at `TEST_PASS` with an `Expansion` nature. + fn row(table: &mut LineageTable, id: NodeId, parents: &[NodeId], blame: &[NodeId]) { + row_tagged( + table, + id, + parents, blame, - nature: Nature::Expansion, - label: "test.transform", - } + RewriteTag { + via: TEST_PASS, + nature: Nature::Expansion, + label: "test.rewrite", + }, + ); } - fn copy(origin: NodeId, produced: Vec, blame: Vec) -> RewriteStep { - RewriteStep { - op: Op::Copy { origin, produced }, - blame, - nature: Nature::Expansion, - label: "test.copy", - } + /// Write one row by hand under a named tag — for the tests where the pass, + /// the nature or the label is the thing under test. + fn row_tagged( + table: &mut LineageTable, + id: NodeId, + parents: &[NodeId], + blame: &[NodeId], + tag: RewriteTag, + ) { + let rule = table.intern_rule(tag); + table.record(id, parents, blame, rule); } - /// One pass' worth of log at `Pass::Inline` (an arbitrary non-lowering pass). - fn phase(steps: Vec) -> Vec<(Pass, LineageLog)> { - vec![(Pass::Inline, steps)] + /// A table holding exactly `rows`, each `(node, parents)` with no blame. + fn table_of(rows: &[(NodeId, &[NodeId])]) -> LineageTable { + let mut table = LineageTable::default(); + for (id, parents) in rows { + row(&mut table, *id, parents, &[]); + } + table } - fn sorted(mut v: Vec) -> Vec { - v.sort_unstable(); - v + /// An attribution holding `spans`, tagged as a lowered direct image — what + /// an upstream pane's entries look like. + fn imaged(spans: &[Span]) -> SourceAttribution { + SourceAttribution { + spans: spans.to_vec(), + rewritten: RewriteTag::direct_image(), + } } // ---- lineage composition ---------------------------------------------- @@ -1412,110 +2069,218 @@ mod tests { fn transient_born_and_consumed_composes_away() { // A → B (born) → C. B exists in neither snapshot; lineage flows A → C. let [a, b, c] = ids(); - let logs = phase(vec![ - transform(vec![a], vec![b], vec![]), - transform(vec![b], vec![c], vec![]), - ]); - let (map, _proj, leaks) = collapse(&logs, &set([a]), &set([c]), &SourceProjection::new()); - assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&c), &[a]); - assert_eq!(map.downstream(&a), &[c]); + let table = table_of(&[(b, &[a]), (c, &[b])]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([c]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!(ids_of(map.upstream(&c)), vec![a]); + assert_eq!(ids_of(map.downstream(&a)), vec![c]); // B is a transient: unknown to the map in both directions. assert!(map.upstream(&b).is_empty()); assert!(map.downstream(&b).is_empty()); } #[test] - fn copy_of_copy_chain_composes() { - // A copied to B, B copied to C; C's roots compose back to A. - let [a, b, c] = ids(); - let logs = phase(vec![copy(a, vec![b], vec![]), copy(b, vec![c], vec![])]); - let (map, _proj, leaks) = - collapse(&logs, &set([a]), &set([a, c]), &SourceProjection::new()); - assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&c), &[a]); - assert_eq!(map.upstream(&a), &[a], "origin keeps its self-edge"); - } - - #[test] - fn copy_whose_origin_is_consumed_later_keeps_edges() { - // Copy A→B, then a Transform consumes A→C. B snapshotted A's lineage - // before A was consumed, so both B and C trace to A. + fn one_parent_reaching_two_products_fans_out() { + // A is rewritten twice — a duplication and a replacement, say. Both + // products trace to A, and A's own fate is decided by the panes alone: + // here it survives, so it keeps its self-edge alongside the fan-out. let [a, b, c] = ids(); - let logs = phase(vec![ - copy(a, vec![b], vec![]), - transform(vec![a], vec![c], vec![]), - ]); - let (map, _proj, leaks) = - collapse(&logs, &set([a]), &set([b, c]), &SourceProjection::new()); + let table = table_of(&[(b, &[a]), (c, &[a])]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([a, b, c]), + &SourceProjection::new(), + ); assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&b), &[a]); - assert_eq!(map.upstream(&c), &[a]); - assert_eq!(sorted(map.downstream(&a).to_vec()), sorted(vec![b, c])); + assert_eq!(ids_of(map.upstream(&b)), vec![a]); + assert_eq!(ids_of(map.upstream(&c)), vec![a]); + assert_eq!( + ids_of(map.upstream(&a)), + vec![a], + "the parent keeps its self-edge" + ); + assert_eq!(sorted(ids_of(map.downstream(&a))), sorted(vec![a, b, c])); } #[test] - fn n_to_m_transform_is_the_bipartite_product() { - // {A,B} → {C,D}: every input reaches every output (2×2 = 4 edges). + fn a_fusion_gives_every_product_the_bipartite_product() { + // {A,B} fused into {C,D}: every input reaches every output (2×2 = 4 + // edges), which falls out of each product row holding the whole + // consumed set as its parents. let [a, b, c, d] = ids(); - let logs = phase(vec![transform(vec![a, b], vec![c, d], vec![])]); - let (map, _proj, leaks) = - collapse(&logs, &set([a, b]), &set([c, d]), &SourceProjection::new()); - assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&c), sorted(vec![a, b]).as_slice()); - assert_eq!(map.upstream(&d), sorted(vec![a, b]).as_slice()); - assert_eq!(map.downstream(&a), sorted(vec![c, d]).as_slice()); - assert_eq!(map.downstream(&b), sorted(vec![c, d]).as_slice()); + let table = table_of(&[(c, &[a, b]), (d, &[a, b])]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a, b]), + &set([c, d]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!(ids_of(map.upstream(&c)), sorted(vec![a, b])); + assert_eq!(ids_of(map.upstream(&d)), sorted(vec![a, b])); + assert_eq!(ids_of(map.downstream(&a)), sorted(vec![c, d])); + assert_eq!(ids_of(map.downstream(&b)), sorted(vec![c, d])); assert_eq!(map.edges().len(), 4); } #[test] fn untouched_id_gets_self_edge_and_inherits_attribution() { - // No steps mention A: it survives with a self-edge and its upstream + // No row mentions A: it survives with a self-edge and its upstream // attribution passes through unchanged. let [a] = ids(); let mut upstream = SourceProjection::new(); - upstream.insert( - a, - SourceAttribution { - spans: vec![span(3, 9)], - rewritten: RewriteTag::direct_image(), - }, + upstream.insert(a, imaged(&[span(3, 9)])); + let (map, proj, leaks) = collapse( + &LineageTable::default(), + PASSES, + &set([a]), + &set([a]), + &upstream, ); - let (map, proj, leaks) = collapse(&phase(vec![]), &set([a]), &set([a]), &upstream); assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&a), &[a]); - assert_eq!(map.downstream(&a), &[a]); + assert_eq!(ids_of(map.upstream(&a)), vec![a]); + assert_eq!(ids_of(map.downstream(&a)), vec![a]); assert_eq!(proj.get(&a), upstream.get(&a), "attribution unchanged"); } + /// A pass set exercising a chain, a fan-out and an N:1 merge, with every + /// parent known and every input dying — so it folds leak-free whichever + /// order its rows were written in. Returns `(rows, inputs, outputs, + /// upstream_attr)` with the rows in dependency order. + #[allow(clippy::type_complexity)] + fn mixed_passes() -> ( + Vec<(NodeId, Vec, Vec)>, + HashSet, + HashSet, + SourceProjection, + ) { + let [a, b, x, y, z] = ids(); + let mut upstream = SourceProjection::new(); + upstream.insert(a, imaged(&[span(0, 2)])); + upstream.insert(b, imaged(&[span(4, 6)])); + let rows = vec![ + (x, vec![a], vec![a]), + (y, vec![x], vec![]), + (z, vec![x, b], vec![b]), + ]; + (rows, set([a, b]), set([y, z]), upstream) + } + + fn table_from(rows: &[(NodeId, Vec, Vec)]) -> LineageTable { + let mut table = LineageTable::default(); + for (id, parents, blame) in rows { + row(&mut table, *id, parents, blame); + } + table + } + #[test] - fn survivor_carry_keeps_own_root_and_absorbs_others() { - // consumed = {A, B}, produced = {A}: A survives while absorbing B. - let [a, b] = ids(); - let logs = phase(vec![transform(vec![a, b], vec![a], vec![])]); - let (map, _proj, leaks) = - collapse(&logs, &set([a, b]), &set([a]), &SourceProjection::new()); - assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&a), sorted(vec![a, b]).as_slice()); + fn the_write_order_of_the_rows_does_not_change_the_fold() { + // The property the ascending sweep buys: the rows are an edge set, so + // writing them in dependency order or in reverse gives byte-identical + // results. Write order is not chronology — rows are written when their + // guard drops, so an enclosing rewrite's rows land after the rows of the + // rewrites nested inside it. + let (rows, inputs, outputs, upstream) = mixed_passes(); + let mut reversed = rows.clone(); + reversed.reverse(); + + let (map, proj, leaks) = collapse(&table_from(&rows), PASSES, &inputs, &outputs, &upstream); + let (rev_map, rev_proj, rev_leaks) = + collapse(&table_from(&reversed), PASSES, &inputs, &outputs, &upstream); + + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!(leaks, rev_leaks, "same leaks in either order"); + assert_eq!( + map.edges(), + rev_map.edges(), + "same relation in either order" + ); + assert_eq!(proj, rev_proj, "same projection in either order"); + } + + #[test] + fn the_sweep_visits_every_vertex_once_and_never_backwards() { + // The sweep's premise, measured: ascending NodeId is a topological order + // of the definition graph, so the revisit count is zero. + let (rows, inputs, _outputs, _upstream) = mixed_passes(); + let m = sweep_metrics(&table_from(&rows), PASSES, &inputs); + assert_eq!(m.vertices, 5, "two input ids + three produced"); + assert_eq!(m.edges, 4, "a→x, x→y, x→z, b→z"); + assert_eq!( + m.backward_edges, 0, + "a backward edge is a vertex the single sweep would have to revisit", + ); + } + + #[test] + fn a_row_produced_outside_the_passes_reads_as_un_produced() { + // The pass restriction, which is what turns a whole-compile table back + // into a per-relation one. B was produced by a pass this relation does + // not span, so to this relation it is an ordinary input-pane node: the + // fold stops there rather than resolving through to A. + let [a, b, c] = ids(); + let mut table = LineageTable::default(); + row_tagged( + &mut table, + b, + &[a], + &[], + RewriteTag { + via: Pass::Mono, + nature: Nature::Machinery, + label: "other.pass", + }, + ); + row(&mut table, c, &[b], &[]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([b]), + &set([c]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!( + ids_of(map.upstream(&c)), + vec![b], + "the out-of-scope row is the relation's input, not a step through it", + ); } // ---- leak taxonomy ----------------------------------------------------- #[test] - fn clean_log_produces_no_leaks() { + fn clean_rows_produce_no_leaks() { let [a, b] = ids(); - let logs = phase(vec![transform(vec![a], vec![b], vec![a])]); - let (_map, _proj, leaks) = collapse(&logs, &set([a]), &set([b]), &SourceProjection::new()); + let table = table_of(&[(b, &[a])]); + let (_map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([a, b]), + &SourceProjection::new(), + ); assert!(leaks.is_empty(), "{leaks:?}"); } #[test] fn leak_unexplained_fires_on_output_with_no_lineage() { - // Z appears in the output snapshot but nothing produced or preserved it. + // Z appears in the output snapshot but no row produced it and the input + // pane does not hold it. let [a, z] = ids(); let (_map, _proj, leaks) = collapse( - &phase(vec![]), + &LineageTable::default(), + PASSES, &set([a]), &set([a, z]), &SourceProjection::new(), @@ -1527,159 +2292,389 @@ mod tests { } #[test] - fn leak_dropped_fires_on_live_input_missing_from_output() { - // B is live at the end (never consumed) but absent from the output. + fn leak_died_fires_on_an_input_missing_from_the_output() { + // B is an input-pane id absent from the output pane, and nothing said so. let [a, b] = ids(); let (_map, _proj, leaks) = collapse( - &phase(vec![]), + &LineageTable::default(), + PASSES, &set([a, b]), &set([a]), &SourceProjection::new(), ); - assert!(leaks.contains(&Leak::Dropped { input: b }), "{leaks:?}"); + assert!(leaks.contains(&Leak::Died { input: b }), "{leaks:?}"); } #[test] - fn leak_consumed_unknown_fires_on_non_live_consume() { + fn leak_parent_unknown_fires_on_a_parent_the_fold_never_heard_of() { + // X is neither an input-pane id nor produced by a pass the fold read, so B's + // lineage stops at an id that describes nothing. One class, whether the + // unknown id is a lone parent (as here) or one of a fusion's several — + // the parents column does not record which. let [a, x, b] = ids(); - let logs = phase(vec![transform(vec![x], vec![b], vec![])]); - let (_map, _proj, leaks) = - collapse(&logs, &set([a]), &set([a, b]), &SourceProjection::new()); + let table = table_of(&[(b, &[x])]); + let (_map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([a, b]), + &SourceProjection::new(), + ); assert!( - leaks.contains(&Leak::ConsumedUnknown { consumed: x }), + leaks.contains(&Leak::ParentUnknown { parent: x }), "{leaks:?}" ); - } - #[test] - fn leak_copy_of_unknown_fires_on_non_live_origin() { - let [a, x, b] = ids(); - let logs = phase(vec![copy(x, vec![b], vec![])]); - let (_map, _proj, leaks) = collapse(&logs, &set([a]), &set([a]), &SourceProjection::new()); + let [a2, x2, b2] = ids(); + let table = table_of(&[(b2, &[a2, x2])]); + let (_map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a2]), + &set([b2]), + &SourceProjection::new(), + ); assert!( - leaks.contains(&Leak::CopyOfUnknown { origin: x }), - "{leaks:?}" + leaks.contains(&Leak::ParentUnknown { parent: x2 }), + "one of a fusion's parents is the same class: {leaks:?}", ); } + // ---- the write-time invariants ----------------------------------------- + // + // Each is a property of the *record*, asserted where the second writer is + // standing rather than in a fold that only runs when a pane is materialized. + // Debug-gated because they are `debug_assert!`s: there is no release + // behaviour to assert, the row being written the same way in every build. + #[test] - fn leak_produced_live_fires_when_producing_an_unconsumed_live_id() { - // Transform consumes A but produces B, which was already live. - let [a, b] = ids(); - let logs = phase(vec![transform(vec![a], vec![b], vec![])]); - let (_map, _proj, leaks) = - collapse(&logs, &set([a, b]), &set([b]), &SourceProjection::new()); - assert!( - leaks.contains(&Leak::ProducedLive { produced: b }), - "{leaks:?}" - ); + #[cfg(debug_assertions)] + fn a_second_row_for_one_id_is_rejected() { + // Attribution has no join — a node cannot carry two `via`/`label` pairs + // — so a second claimant has no answer and overwriting would make the + // survivor a lie about which rewrite made the node. + let [a, b, p] = ids(); + let caught = std::panic::catch_unwind(|| { + let mut table = LineageTable::default(); + row(&mut table, p, &[a], &[]); + row(&mut table, p, &[b], &[]); + }); + assert!(caught.is_err(), "a duplicate produce must be rejected"); } #[test] - fn empty_consumed_with_blame_is_the_legal_pure_insertion_shape() { - // A consume-nothing Transform that names blame is a pure insertion: the - // output gets empty lineage roots (present in the map with empty `up`) - // and a blame-attributed projection entry. No EmptyConsumed leak. - let [a, b] = ids(); - let mut upstream = SourceProjection::new(); - upstream.insert( - a, - SourceAttribution { - spans: vec![span(1, 4)], - rewritten: RewriteTag::direct_image(), - }, - ); - let logs = phase(vec![transform(vec![], vec![b], vec![a])]); - let (map, proj, leaks) = collapse(&logs, &set([a]), &set([a, b]), &upstream); - assert!(leaks.is_empty(), "pure insertion is leak-free: {leaks:?}"); - // b is present in the map with genuinely empty lineage roots. - assert!( - map.upstream(&b).is_empty(), - "pure insertion has no lineage ancestor" - ); - // b is attributed via its blame, not via consumption. - let attr = proj.get(&b).expect("b attributed via blame"); - assert_eq!(attr.spans, vec![span(1, 4)]); + #[cfg(debug_assertions)] + fn a_row_with_neither_parent_nor_blame_is_rejected() { + // A truly unanchored mint: nothing explains where the node came from. + let [b] = ids(); + let caught = std::panic::catch_unwind(|| { + let mut table = LineageTable::default(); + row(&mut table, b, &[], &[]); + }); + assert!(caught.is_err(), "an unanchored record must be rejected"); } #[test] - fn leak_empty_consumed_fires_when_consumed_and_blame_both_empty() { - // A truly unanchored mint: neither consumption nor blame explains it. + #[cfg(debug_assertions)] + fn a_node_may_not_be_its_own_parent() { + // The self-referential definition — the one construct that makes an edge + // run backwards and forces the fold to a fixed point. An in-place + // rewrite that keeps its id is a preserve: it records nothing. let [a, b] = ids(); - let logs = phase(vec![transform(vec![], vec![b], vec![])]); - let (_map, _proj, leaks) = - collapse(&logs, &set([a]), &set([a, b]), &SourceProjection::new()); - assert!( - leaks - .iter() - .any(|l| matches!(l, Leak::EmptyConsumed { .. })), - "{leaks:?}" - ); + let caught = std::panic::catch_unwind(|| { + let mut table = LineageTable::default(); + row(&mut table, a, &[a, b], &[]); + }); + assert!(caught.is_err(), "consumed ∩ produced ≠ ∅ must be rejected"); } // ---- blame / attribution ---------------------------------------------- #[test] - fn expansion_blame_yields_deduped_ordered_span_union() { - // blame = [A, B]; A: [s1, s2], B: [s2, s3] → [s1, s2, s3]. + fn a_pure_insertion_is_blamed_and_descends_from_nothing() { + // A node placed over surviving material: no parents at all, attributed + // through blame. Its one edge is the blame edge blame contributes; + // nothing claims it descends from anything. + let [a, b] = ids(); + let mut upstream = SourceProjection::new(); + upstream.insert(a, imaged(&[span(1, 4)])); + let mut table = LineageTable::default(); + row(&mut table, b, &[], &[a]); + let (map, proj, leaks) = collapse(&table, PASSES, &set([a]), &set([a, b]), &upstream); + assert!(leaks.is_empty(), "pure insertion is leak-free: {leaks:?}"); + assert_eq!( + labels_of(map.upstream(&b), a), + Some(EdgeLabels::BLAME), + "the insertion is related to what it was blamed on, and descends from nothing", + ); + assert_eq!(ids_of(map.upstream(&b)), vec![a], "and from nothing else"); + let attr = proj.get(&b).expect("b attributed via blame"); + assert_eq!(attr.spans, vec![span(1, 4)]); + } + + #[test] + fn the_span_union_is_deduplicated_across_both_channels() { + // parents = [A], blame = [A, B] — A is named twice and A's own spans + // overlap B's. A: [s1, s2], B: [s2, s3] → [s1, s2, s3], each span once. let [a, b, out] = ids(); let (s1, s2, s3) = (span(0, 1), span(2, 3), span(4, 5)); let mut upstream = SourceProjection::new(); - upstream.insert( - a, - SourceAttribution { - spans: vec![s1, s2], - rewritten: RewriteTag::direct_image(), - }, + upstream.insert(a, imaged(&[s1, s2])); + upstream.insert(b, imaged(&[s2, s3])); + let mut table = LineageTable::default(); + row(&mut table, out, &[a], &[a, b]); + // B is carried to the output pane so the fold reports no death for it. + let (_map, proj, leaks) = collapse(&table, PASSES, &set([a, b]), &set([out, b]), &upstream); + assert_eq!( + defects(&leaks), + Vec::<&Leak>::new(), + "blame does not affect fate accounting: {leaks:?}" ); - upstream.insert( + let attr = proj.get(&out).expect("out attributed"); + assert_eq!(attr.spans, vec![s1, s2, s3]); + assert_eq!(attr.rewritten.via, Pass::Inline); + assert_eq!(attr.rewritten.nature, Nature::Expansion); + } + + #[test] + fn empty_blame_attributes_through_the_parents() { + // The common case: no blame, so the node's spans are its parent's, + // re-tagged with the rewrite that produced it. + let [a, b] = ids(); + let mut upstream = SourceProjection::new(); + upstream.insert(a, imaged(&[span(1, 2)])); + let mut table = LineageTable::default(); + row_tagged( + &mut table, b, - SourceAttribution { - spans: vec![s2, s3], - rewritten: RewriteTag::direct_image(), + &[a], + &[], + RewriteTag { + via: TEST_PASS, + nature: Nature::Expansion, + label: "copy.mirror", }, ); - // A and B survive (blame ⊥ consumption); the step consumes neither but - // carries them, so use a survivor-carry-free shape: consume a separate - // transient. Here we consume A and produce `out`, blaming both A and B. - let logs = phase(vec![transform(vec![a], vec![out], vec![a, b])]); - let (_map, proj, leaks) = collapse(&logs, &set([a, b]), &set([out, b]), &upstream); - // B is carried to the output to avoid a Dropped leak; A is consumed. + let (_map, proj, leaks) = collapse(&table, PASSES, &set([a]), &set([a, b]), &upstream); + assert!(leaks.is_empty(), "{leaks:?}"); + let attr = proj.get(&b).expect("product attributed"); + assert_eq!(attr.spans, vec![span(1, 2)], "mirrors the parent's spans"); + assert_eq!(attr.rewritten.via, Pass::Inline); + assert_eq!(attr.rewritten.nature, Nature::Expansion); + assert_eq!(attr.rewritten.label, "copy.mirror"); + } + + #[test] + fn empty_blame_on_a_fusion_unions_every_parents_spans() { + // The many:1 case of the same rule, and the reason it is one rule: a + // fusion names no blame, and its product is an image of everything it + // was made from — so it resolves to the union, not to nothing. + let [a, b, out] = ids(); + let mut upstream = SourceProjection::new(); + upstream.insert(a, imaged(&[span(0, 3)])); + upstream.insert(b, imaged(&[span(7, 9)])); + let table = table_of(&[(out, &[a, b])]); + let (_map, proj, leaks) = collapse(&table, PASSES, &set([a, b]), &set([out]), &upstream); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); assert_eq!( - leaks, - vec![], - "blame does not affect fate accounting: {leaks:?}" + proj.get(&out).expect("fusion attributed").spans, + vec![span(0, 3), span(7, 9)], + ); + } + + #[test] + fn parents_and_blame_reach_the_relation_under_different_labels() { + // The two channels populated with *distinct* nodes: the product was made + // from P and is additionally *about* B. Its spans are both, parentage + // first. Both nodes reach the relation, and the labels are what keep the + // claims apart — B survives the rewrite, so "the product is related to + // B" must not read as "the product descends from B". + let [p, b, out] = ids(); + let (sp, sb) = (span(0, 4), span(9, 12)); + let mut upstream = SourceProjection::new(); + upstream.insert(p, imaged(&[sp])); + upstream.insert(b, imaged(&[sb])); + let mut table = LineageTable::default(); + row(&mut table, out, &[p], &[b]); + let (map, proj, leaks) = collapse(&table, PASSES, &set([p, b]), &set([b, out]), &upstream); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!( + proj.get(&out).expect("out attributed").spans, + vec![sp, sb], + "both channels resolve, parentage first", + ); + assert_eq!( + labels_of(map.upstream(&out), p), + Some(EdgeLabels::ANCESTRY), + "ancestry-only: the consumed node is an ancestor and nothing else", + ); + assert_eq!( + labels_of(map.upstream(&out), b), + Some(EdgeLabels::BLAME), + "blame-only: the blamed node is named, not descended from", + ); + assert_eq!( + labels_of(map.downstream(&b), b), + Some(EdgeLabels::ANCESTRY), + "and B, surviving, still descends from itself", + ); + } + + #[test] + fn one_id_in_both_columns_is_one_edge_carrying_both_labels() { + // A rewrite that consumes P *and* names it as blame — the case the + // per-pair storage exists for. It is one pair, so it is one edge, and + // the edge asserts both relations rather than the fold picking a winner + // or the map holding the pair twice. + let [p, out] = ids(); + let mut table = LineageTable::default(); + row(&mut table, out, &[p], &[p]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([p]), + &set([out]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!( + ids_of(map.upstream(&out)), + vec![p], + "one entry for one pair" + ); + let labels = labels_of(map.upstream(&out), p).expect("the pair is an edge"); + assert!(labels.has_ancestry() && labels.has_blame(), "{labels:?}"); + } + + #[test] + fn two_paths_to_one_root_union_their_labels() { + // The same pair reached twice, once each way: OUT descends from X, which + // descends from R, and OUT is separately blamed on R. Both readings are + // true of the pair `(R, OUT)`, and the entry carries both — a consumer + // pruning blame still sees the ancestry, and one pruning ancestry + // still sees the blame. + let [r, x, out] = ids(); + let mut table = LineageTable::default(); + row(&mut table, x, &[r], &[]); + row(&mut table, out, &[x], &[r]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([r]), + &set([out]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + let labels = labels_of(map.upstream(&out), r).expect("the pair is an edge"); + assert!(labels.has_ancestry() && labels.has_blame(), "{labels:?}"); + } + + #[test] + fn a_mixed_path_is_blame_not_ancestry() { + // Weakest link, which is the whole content of the label. OUT descends + // from M, and M is *related to* R — so OUT is related to R and does not + // descend from it. Reading the closure as unlabelled reachability would + // make R an ancestor of OUT while R is still standing in the tree. + let [r, m, out] = ids(); + let mut table = LineageTable::default(); + row(&mut table, m, &[], &[r]); + row(&mut table, out, &[m], &[]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([r]), + &set([r, out]), + &SourceProjection::new(), + ); + assert!(leaks.is_empty(), "{leaks:?}"); + assert_eq!( + labels_of(map.upstream(&out), r), + Some(EdgeLabels::BLAME), + "one blame hop on the path makes the endpoint related", + ); + assert_eq!( + labels_of(map.downstream(&r), out), + Some(EdgeLabels::BLAME), + "and the mirrored direction agrees", + ); + } + + #[test] + fn an_all_ancestry_path_stays_ancestry_through_a_transient() { + // The other half of weakest-link: composing ancestry with ancestry is + // ancestry however many transients the path runs through, so the label + // is not merely "one hop, unrewritten". + let [a, b, c] = ids(); + let table = table_of(&[(b, &[a]), (c, &[b])]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([c]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!(labels_of(map.upstream(&c), a), Some(EdgeLabels::ANCESTRY)); + } + + #[test] + fn a_blamed_id_the_fold_never_heard_of_is_not_a_parent_unknown() { + // `ParentUnknown` is a claim about the `parents` column: an *ancestry* hop + // that stops at an id describing nothing. Blame points at material the + // relation need not hold, so an unknown blamed id contributes no edge and + // no leak — the same silence `attribute` keeps for a blamed id with no + // known spans. + let [a, unknown, b] = ids(); + let mut table = LineageTable::default(); + row(&mut table, b, &[a], &[unknown]); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([b]), + &SourceProjection::new(), + ); + assert_eq!( + defects(&leaks), + Vec::<&Leak>::new(), + "an unknown blamed id is not a defect: {leaks:?}" + ); + assert_eq!( + ids_of(map.upstream(&b)), + vec![a], + "and it contributes no edge" ); - let attr = proj.get(&out).expect("out attributed"); - assert_eq!(attr.spans, vec![s1, s2, s3]); - let tag = &attr.rewritten; - assert_eq!(tag.via, Pass::Inline); - assert_eq!(tag.nature, Nature::Expansion); } #[test] fn machinery_empty_blame_is_present_with_empty_spans() { // The "known node, no source anchor" case: present in the projection - // with spans: [], distinct from a node absent from the projection. + // with spans: [], distinct from a node absent from the projection. Its + // ancestor has no spans either, so there is nothing to inherit. let [a, b] = ids(); - let logs = vec![( - Pass::Inline, - vec![RewriteStep { - op: Op::Transform { - consumed: vec![a], - produced: vec![b], - }, - blame: vec![], + let mut table = LineageTable::default(); + row_tagged( + &mut table, + b, + &[a], + &[], + RewriteTag { + via: TEST_PASS, nature: Nature::Machinery, label: "machinery.plumbing", - }], - )]; - let (_map, proj, leaks) = collapse(&logs, &set([a]), &set([b]), &SourceProjection::new()); - assert!(leaks.is_empty(), "{leaks:?}"); + }, + ); + let (_map, proj, leaks) = collapse( + &table, + PASSES, + &set([a]), + &set([b]), + &SourceProjection::new(), + ); + assert!(defects(&leaks).is_empty(), "{leaks:?}"); let attr = proj.get(&b).expect("b present in projection"); assert!(attr.spans.is_empty(), "known node, no source anchor"); - let tag = &attr.rewritten; - assert_eq!(tag.nature, Nature::Machinery); + assert_eq!(attr.rewritten.nature, Nature::Machinery); } #[test] @@ -1688,7 +2683,8 @@ mod tests { // no projection entry at all — distinct from present-but-empty spans. let [a] = ids(); let (_map, proj, leaks) = collapse( - &phase(vec![]), + &LineageTable::default(), + PASSES, &set([a]), &set([a]), &SourceProjection::new(), @@ -1700,68 +2696,25 @@ mod tests { ); } - #[test] - fn copy_empty_blame_mirrors_origin_retagged() { - // Copy with empty blame mirrors the origin's spans, re-tagged with the - // copy's {via, nature, label}. - let [a, b] = ids(); - let mut upstream = SourceProjection::new(); - upstream.insert( - a, - SourceAttribution { - spans: vec![span(1, 2)], - rewritten: RewriteTag::direct_image(), - }, - ); - let logs = vec![( - Pass::Inline, - vec![RewriteStep { - op: Op::Copy { - origin: a, - produced: vec![b], - }, - blame: vec![], - nature: Nature::Expansion, - label: "copy.mirror", - }], - )]; - let (_map, proj, leaks) = collapse(&logs, &set([a]), &set([a, b]), &upstream); - assert!(leaks.is_empty(), "{leaks:?}"); - let attr = proj.get(&b).expect("copy attributed"); - assert_eq!(attr.spans, vec![span(1, 2)], "mirrors origin spans"); - let tag = &attr.rewritten; - assert_eq!(tag.via, Pass::Inline); - assert_eq!(tag.nature, Nature::Expansion); - assert_eq!(tag.label, "copy.mirror"); - } - // ---- the lowering fold (collapse_lowering) ----------------------------- fn leaf(id: NodeId, sp: Span, nature: Nature, label: RewriteLabel) -> LoweringStep { - LoweringStep { - op: Op::Transform { - consumed: Vec::new(), - produced: vec![id], - }, - anchor: vec![sp], + LoweringStep::Leaf { + id, + anchor: sp, nature, label, } } fn lowering_copy(origin: NodeId, produced: Vec) -> LoweringStep { - LoweringStep { - op: Op::Copy { origin, produced }, - anchor: Vec::new(), - nature: Nature::Source, - label: "lower.copy", - } + LoweringStep::Copy { origin, produced } } #[test] - fn lowering_pure_insertion_leaf_is_attributed_from_its_anchor() { - // A leaf mint: no input pane, empty roots, attribution straight from the - // literal anchor span with the direct-image tag. + fn lowering_leaf_is_attributed_from_its_anchor() { + // A leaf mint: no input pane, no lineage ancestor, attribution straight + // from the literal anchor span with the direct-image tag. let [a] = ids(); let log = vec![leaf(a, span(2, 7), Nature::Source, "lower.image")]; let (proj, leaks) = collapse_lowering(&log, &set([a])); @@ -1800,9 +2753,9 @@ mod tests { fn lowering_born_copied_discarded_template_composes_away() { // The uncurry shape: a template proj node is minted (leaf, Machinery), // copied into an occurrence's interior, and never itself placed in the - // output tree. It is live at the end but neither placed nor an output id, - // so it composes away with NO leak (there is no Dropped class in - // lowering, and Unexplained checks outputs only). + // output tree. It is live at the end but not an output id, so it + // composes away with NO leak (there is no Died class in lowering, and + // Unexplained checks outputs only). let [template, occ_interior] = ids(); let log = vec![ leaf( @@ -1844,11 +2797,23 @@ mod tests { } #[test] - fn lowering_substituted_root_inherits_its_occurrences_leaf_attribution() { - // A substituted root carries the occurrence's own id, so the log holds no - // step for it — just the occurrence's leaf entry — while its interior - // children are copies of the replacement template. The root keeps its own - // (Source) attribution; the interior mirrors the template (Machinery). + fn lowering_parent_unknown_fires_on_a_copy_of_an_unrecorded_origin() { + let [never, copy] = ids(); + let log = vec![lowering_copy(never, vec![copy])]; + let (_proj, leaks) = collapse_lowering(&log, &set([copy])); + assert!( + leaks.contains(&Leak::ParentUnknown { parent: never }), + "{leaks:?}" + ); + } + + #[test] + fn lowering_root_carry_preserve_inherits_its_leaf_attribution() { + // Root-carry: the substituted compound root carries the + // occurrence's own id (a preserve). In the log that is just the + // occurrence's leaf entry — no copy for the root — and its interior + // children are copies of the template. The root keeps its own (Source) + // attribution; the interior mirrors the template (Machinery). let [occurrence, tmpl_child, occ_child] = ids(); let log = vec![ // The param-use occurrence, imaged Source at its mint. @@ -1881,11 +2846,12 @@ mod tests { } #[test] - fn lowering_reimage_last_tag_wins_without_produced_live_leak() { + fn lowering_reimage_last_tag_wins() { // `lower_expr` re-tags an arm's already-tagged root as the construct's - // direct image: two leaf steps for one id. The lowering fold treats the - // re-image as last-tag-wins (no ProducedLive leak); the later Source tag - // wins over the earlier machinery one. + // direct image: two leaf records for one id. Lowering has no + // one-record-per-id rule (the fold is sequential and its log genuinely + // is chronology), so the later Source tag wins over the earlier + // machinery one, with no leak. let [id] = ids(); let log = vec![ leaf(id, span(0, 5), Nature::Machinery, "lower.compare_chain"), @@ -1902,294 +2868,445 @@ mod tests { // ---- the recorder ------------------------------------------------------ // - // These exercise the hooks through *real* `Expr` construction (`Expr::lit`, - // `Expr::tuple`) and the freshening `Clone`, not hand-built steps, so the hook - // wiring in `expr.rs` is under test too. + // These exercise the construction hooks through *real* `Expr` construction + // (`Expr::new`/`Expr::lit`/`Expr::tuple` + a freshening `Clone`), not + // hand-written rows, so the hook wiring in `expr.rs` is under test too. - use crate::ccl::Lit; use crate::ccl::expr::Expr; + use crate::ccl::{Lit, TypedExprNode}; - #[test] - fn transform_step_captures_births_as_produced() { - let session = RecorderSession::new(); - let consumed = NodeId::fresh(); - let (a, b); + /// Run `f` with a table installed and `TEST_PASS` ambient, and return the + /// rows it recorded — the two things a pass boundary sets up. + fn recorded(f: impl FnOnce()) -> LineageTable { + let table = TableSession::install(); { - let _g = step("rw.build", vec![consumed], vec![], Nature::Expansion); - a = Expr::lit(Lit::Int(1)).node_id(); - b = Expr::lit(Lit::Int(2)).node_id(); + let _scope = PassScope::enter(TEST_PASS); + f(); } + table.into_table() + } + + /// The predicate sweep must not overwrite attribution a node already has. + /// + /// This is the one property of `lowering_predicate_leaf` that **no leak class + /// can see**: the fold is last-write-wins, so a node re-recorded by the sweep + /// is still perfectly *explained* — it has just silently swapped its real + /// span and label for the sweep's coarse ones. Measured on the pipeline + /// corpus, a blanket sweep clobbers 318 nodes and every gate stays green. + #[test] + fn the_predicate_sweep_skips_already_recorded_nodes() { + let [recorded_id, fresh] = ids::<2>(); + let session = LoweringSession::install(); + lowering_leaf(recorded_id, span(10, 20), Nature::Source, "lower.precise"); + lowering_predicate_leaf(recorded_id, span(0, 99), Nature::Machinery, "lower.sweep"); + lowering_predicate_leaf(fresh, span(0, 99), Nature::Machinery, "lower.sweep"); let log = session.into_log(); - assert_eq!(log.len(), 1); + + assert_eq!(log.len(), 2, "the already-recorded node is not re-recorded"); + let leaf = |want: NodeId| { + log.iter() + .find_map(|s| match s { + LoweringStep::Leaf { + id, anchor, label, .. + } if *id == want => Some((*anchor, *label)), + _ => None, + }) + .expect("a leaf for this id") + }; assert_eq!( - log[0].op, - Op::Transform { - consumed: vec![consumed], - produced: vec![a, b], - }, - "produced = exactly the ids built in the step's extent, in order", + leaf(recorded_id), + (span(10, 20), "lower.precise"), + "the lowered node keeps its own span and label", + ); + assert_eq!( + leaf(fresh), + (span(0, 99), "lower.sweep"), + "the assembly node is explained by the sweep", ); - assert_eq!(step_stack_depth(), 0, "guard popped its frame"); } + /// A second sweep over the same predicate adds nothing — the skip is keyed on + /// the id, so overlapping predicates (one term riding several type slots) + /// cannot double-record. #[test] - fn nested_steps_capture_births_in_the_innermost_frame_only() { - let session = RecorderSession::new(); - let (outer_c, inner_c) = (NodeId::fresh(), NodeId::fresh()); - let (outer_pre, inner_id, outer_post); - { - let _outer = step("rw.outer", vec![outer_c], vec![], Nature::Expansion); + fn the_predicate_sweep_is_idempotent() { + let [n] = ids::<1>(); + let session = LoweringSession::install(); + lowering_predicate_leaf(n, span(1, 2), Nature::Machinery, "lower.sweep"); + lowering_predicate_leaf(n, span(3, 4), Nature::Machinery, "lower.sweep"); + let log = session.into_log(); + assert_eq!( + log.len(), + 1, + "one entry per node however many sweeps reach it" + ); + assert!(matches!(&log[0], LoweringStep::Leaf { anchor, .. } if *anchor == span(1, 2))); + } + + #[test] + fn a_recording_rows_every_birth_on_its_slot() { + let slot = NodeId::fresh(); + let (mut a, mut b) = (NodeId::PLACEHOLDER, NodeId::PLACEHOLDER); + let table = recorded(|| { + let _g = enter(slot, "rw.build", Nature::Expansion); + a = Expr::lit(Lit::Int(1)).node_id(); + b = Expr::lit(Lit::Int(2)).node_id(); + }); + assert_eq!( + table.parents(a), + &[slot], + "a birth is parented on the node the recording named", + ); + assert_eq!(table.parents(b), &[slot]); + assert_eq!( + table.rule(a), + Some(RewriteTag { + via: TEST_PASS, + nature: Nature::Expansion, + label: "rw.build", + }), + "the row's `via` is the ambient pass — the one part of the tag no \ + recording site knows", + ); + assert_eq!(table.rule(a), table.rule(b)); + assert!( + !table.contains(slot), + "the named slot is read, not produced: it gets no row of its own", + ); + assert_eq!(table.len(), 2, "one row per produced id"); + assert_eq!(step_stack_depth(), 0, "guard popped its recording"); + } + + #[test] + fn nested_recordings_attribute_each_mint_to_its_innermost_slot() { + // Granularity is precision: a mint attributes to the innermost open + // recording. A coarser recording is not *wrong*, it is less precise — + // the mint attaches to whatever enclosing node was named. + let (outer_slot, inner_slot) = (NodeId::fresh(), NodeId::fresh()); + let (mut outer_pre, mut inner_id, mut outer_post) = ( + NodeId::PLACEHOLDER, + NodeId::PLACEHOLDER, + NodeId::PLACEHOLDER, + ); + let table = recorded(|| { + let _outer = enter(outer_slot, "rw.outer", Nature::Expansion); outer_pre = Expr::lit(Lit::Int(0)).node_id(); { - let _inner = step("rw.inner", vec![inner_c], vec![], Nature::Expansion); + let _inner = enter(inner_slot, "rw.inner", Nature::Expansion); inner_id = Expr::lit(Lit::Int(1)).node_id(); } outer_post = Expr::lit(Lit::Int(2)).node_id(); - } - let log = session.into_log(); - // Inner flushes on its (earlier) drop, then outer. - assert_eq!(log.len(), 2); - assert_eq!( - log[0].op, - Op::Transform { - consumed: vec![inner_c], - produced: vec![inner_id], - }, - "inner step owns only the birth in its extent", - ); + }); + assert_eq!(table.parents(inner_id), &[inner_slot]); + assert_eq!(table.parents(outer_pre), &[outer_slot]); assert_eq!( - log[1].op, - Op::Transform { - consumed: vec![outer_c], - produced: vec![outer_pre, outer_post], - }, - "outer step owns the births outside the inner extent, not the inner birth", + table.parents(outer_post), + &[outer_slot], + "the outer recording owns the births outside the inner extent, and only those", ); + assert_eq!(table.rule(inner_id).map(|t| t.label), Some("rw.inner")); + assert_eq!(table.rule(outer_pre).map(|t| t.label), Some("rw.outer")); } #[test] - fn deep_freshen_in_a_step_yields_per_origin_copies() { - // Build a 3-node tree (tuple + two lits) OUTSIDE any step, then clone it - // inside a copy-only frame — one that consumes nothing and mints nothing - // of its own, so its whole output is the freshen pairs `Clone` reports - // through the `on_copy` hook. + fn a_deep_freshen_rows_each_node_on_its_own_origin() { + // Build a 3-node tree (tuple + two lits) with nothing recording, then + // clone it inside a copy-only recording — one that consumes nothing and + // mints nothing of its own, so its whole output is the freshen pairs + // `Clone` reports through the `on_copy` hook. let source = Expr::tuple(vec![Expr::lit(Lit::Int(1)), Expr::lit(Lit::Int(2))]); - // The source's node ids are the origins each per-node `Copy` should record. + // The source's node ids are the origins each per-node row should name. let old_ids: HashSet = std::iter::once(source.node_id()) .chain(source.child_exprs().iter().map(|c| c.node_id())) .collect(); - let session = RecorderSession::new(); - let clone = { + let mut clone = Expr::lit(Lit::Int(0)); + let table = recorded(|| { let _g = copy_frame("dup"); - source.clone() - }; - let log = session.into_log(); + clone = source.clone(); + }); - // The clone is a distinct 3-node tree sharing no id with its source. let fresh_ids: HashSet = std::iter::once(clone.node_id()) .chain(clone.child_exprs().iter().map(|c| c.node_id())) .collect(); - assert_eq!(fresh_ids.len(), 3); + assert_eq!(fresh_ids.len(), 3, "the clone is a distinct 3-node tree"); assert!( fresh_ids.is_disjoint(&old_ids), - "a clone shares no id with its source" - ); - - // Three nodes freshened: three per-origin Copy steps, one produced each, - // one Copy per pre-freshen origin id. - assert_eq!(log.len(), 3, "one Copy step per freshened origin"); - let mut copied_origins = HashSet::new(); - for s in &log { - match &s.op { - Op::Copy { origin, produced } => { - assert_eq!(produced.len(), 1, "one produced entry per freshened node"); - copied_origins.insert(*origin); - } - other => panic!("expected only Copy steps, got {other:?}"), - } + "a clone shares no id with its source", + ); + assert_eq!(table.len(), 3, "one row per cloned node"); + let mut origins: HashSet = HashSet::new(); + for id in fresh_ids { + let parents = table.parents(id); + assert_eq!(parents.len(), 1, "a copy has exactly its origin as parent"); + origins.insert(parents[0]); } assert_eq!( - copied_origins, old_ids, - "every freshened node recorded a per-origin copy", + origins, old_ids, + "every cloned node rows on the node it was duplicated from", ); } #[test] - fn transform_frame_capturing_only_freshens_emits_no_empty_transform() { - // A Transform frame opened purely to capture a deep freshen (no consumed - // ids, no births) emits only its per-origin Copy steps — never an empty - // `Transform { consumed: [], produced: [] }`. + fn a_recording_that_only_clones_records_nothing_of_its_own() { + // A recording whose rewrite turns out to be a pure duplication (nothing + // minted by hand, nothing fused) writes only the rows the clone reported, + // none of them naming its slot. let source = Expr::tuple(vec![Expr::lit(Lit::Int(1)), Expr::lit(Lit::Int(2))]); - - let session = RecorderSession::new(); + // The named slot is the node being rewritten, not the tree being + // duplicated — the two are distinct at every real site. + let slot = NodeId::fresh(); + let mut clone = Expr::lit(Lit::Int(0)); + let table = recorded(|| { + let _g = enter(slot, "wrap.freshen", Nature::Machinery); + clone = source.clone(); + }); + assert_eq!(table.len(), 3, "only the three cloned nodes"); + for id in + std::iter::once(clone.node_id()).chain(clone.child_exprs().iter().map(|c| c.node_id())) { - let _g = step("wrap.freshen", vec![], vec![], Nature::Machinery); - let _clone = source.clone(); + assert_ne!( + table.parents(id), + &[slot], + "each row names the cloned node's own origin, not the recording's slot", + ); } - let log = session.into_log(); - assert_eq!(log.len(), 3, "only the three per-origin Copy steps"); + } + + #[test] + fn a_captured_freshen_rows_on_its_own_origin() { + // Most production in `channelize`/`transact_phase` is a `clone`, not + // an `Expr::new`. Those fire `on_copy`, whose origin is the *copied* node, + // not the named slot — so the copy channel stays independent of the + // recording's own parentage rather than being folded into it. + let tree = Expr::tuple(vec![Expr::lit(Lit::Int(1))]); + let slot = NodeId::fresh(); + let mut copy_root = NodeId::PLACEHOLDER; + let table = recorded(|| { + let _g = enter(slot, "rw.duplicate", Nature::Machinery); + copy_root = tree.clone().node_id(); + }); + assert_eq!(table.parents(copy_root), &[tree.node_id()]); assert!( - log.iter().all(|s| matches!(s.op, Op::Copy { .. })), - "no empty Transform emitted: {log:?}", + table.blame(copy_root).is_empty(), + "a copy mirrors its origin rather than re-attributing", ); } #[test] - fn no_step_open_records_nothing() { - let session = RecorderSession::new(); - // Construction and cloning with an empty stack capture nowhere. - let _e = Expr::lit(Lit::Int(1)); - let x = Expr::lit(Lit::Int(2)); - let _copy = x.clone(); - let log = session.into_log(); - assert!(log.is_empty(), "empty stack ⇒ nothing recorded: {log:?}"); + fn no_frame_open_records_nothing() { + let table = recorded(|| { + // Construction and cloning with an empty stack capture nowhere. + let _e = Expr::lit(Lit::Int(1)); + let x = Expr::lit(Lit::Int(2)); + let _copy = x.clone(); + }); + assert_eq!(table.len(), 0, "empty stack ⇒ nothing recorded"); } #[test] - fn no_session_installed_makes_the_flush_a_silent_no_op() { - // A step open with no ACTIVE_LOG: births still capture into the frame, - // but the guard's flush finds no log and does nothing (no panic). + fn no_pass_scope_open_makes_the_flush_a_silent_no_op() { + // A recording open with no ambient pass: births still capture into it, + // but the guard has no tag to complete a row with and writes nothing (no + // panic). assert!( - ACTIVE_LOG.with(|s| s.borrow().is_none()), - "precondition: no session installed", + ACTIVE_PASS.with(|s| s.borrow().is_none()), + "precondition: no pass scope open", ); + let table_session = TableSession::install(); { - let _g = step("rw", vec![NodeId::fresh()], vec![], Nature::Expansion); + let _g = enter(NodeId::fresh(), "rw", Nature::Expansion); let _a = Expr::lit(Lit::Int(1)); } assert_eq!( step_stack_depth(), 0, - "guard still pops cleanly with no log" + "guard still pops cleanly with no pass scope" + ); + assert_eq!(table_session.into_table().len(), 0); + } + + #[test] + fn no_table_installed_makes_the_flush_a_silent_no_op() { + assert!( + ACTIVE_TABLE.with(|s| s.borrow().is_none()), + "precondition: no table installed", ); + let _scope = PassScope::enter(TEST_PASS); + let slot = NodeId::fresh(); + { + let _g = enter(slot, "rw.build", Nature::Machinery); + let _ = Expr::lit(Lit::Int(1)); + } + assert_eq!(step_stack_depth(), 0, "guard still pops cleanly"); } #[test] - fn panic_inside_a_step_unwinds_without_poisoning_the_stack() { + fn panic_inside_a_recording_unwinds_without_poisoning_the_stack() { assert_eq!(step_stack_depth(), 0, "clean precondition"); let result = std::panic::catch_unwind(|| { - let _g = step("boom", vec![NodeId::fresh()], vec![], Nature::Expansion); + let _g = enter(NodeId::fresh(), "boom", Nature::Expansion); let _a = Expr::lit(Lit::Int(1)); - panic!("deliberate panic inside an open step"); + panic!("deliberate panic inside an open recording"); }); assert!(result.is_err(), "the panic propagated"); assert_eq!( step_stack_depth(), 0, - "the guard's Drop popped the frame on unwind" + "the guard's Drop popped the recording on unwind" ); // Recording still works afterward. - let session = RecorderSession::new(); - let consumed = NodeId::fresh(); - let a; - { - let _g = step("rw", vec![consumed], vec![], Nature::Expansion); + let slot = NodeId::fresh(); + let mut a = NodeId::PLACEHOLDER; + let table = recorded(|| { + let _g = enter(slot, "rw", Nature::Expansion); a = Expr::lit(Lit::Int(7)).node_id(); - } - let log = session.into_log(); - assert_eq!(log.len(), 1); - assert_eq!( - log[0].op, - Op::Transform { - consumed: vec![consumed], - produced: vec![a], - }, - ); + }); + assert_eq!(table.parents(a), &[slot]); } #[test] fn default_uses_placeholder_records_nothing_and_two_defaults_share_id() { - let session = RecorderSession::new(); - let d1 = Expr::default(); - let d2 = Expr::default(); - // Even inside an open step, a Default throwaway must not be captured. - let consumed = NodeId::fresh(); - { - let _g = step("rw", vec![consumed], vec![], Nature::Expansion); + let slot = NodeId::fresh(); + let (mut d1, mut d2) = (NodeId::PLACEHOLDER, NodeId::PLACEHOLDER); + let mut kept = NodeId::PLACEHOLDER; + let table = recorded(|| { + d1 = Expr::default().node_id(); + d2 = Expr::default().node_id(); + // Even inside an open recording, a Default throwaway must not be + // captured. + let _g = enter(slot, "rw", Nature::Expansion); let _d3 = Expr::default(); - } - let log = session.into_log(); + // A real mint alongside it, so something is recorded at all: a + // recording that captures nothing is a preserve and stays silent, + // which would make "the Default was not captured" vacuously true. + kept = Expr::lit(Lit::Int(3)).node_id(); + }); + assert_eq!(d1, NodeId::PLACEHOLDER, "Default mints the sentinel"); assert_eq!( - d1.node_id(), - NodeId::PLACEHOLDER, - "Default mints the sentinel" - ); - assert_eq!( - d1.node_id(), - d2.node_id(), + d1, d2, "two Defaults share the placeholder id (they compare equal regardless — \ node_id is excluded from PartialEq)", ); - assert_eq!(log.len(), 1); - match &log[0].op { - Op::Transform { produced, .. } => { - assert!( - produced.is_empty(), - "the Default was not captured: {produced:?}" - ) - } - other => panic!("expected a Transform, got {other:?}"), - } + assert_eq!(table.len(), 1, "the Default throwaway got no row"); + assert_eq!(table.parents(kept), &[slot], "the real mint was captured"); + } + + #[test] + fn a_fused_flush_gives_its_product_every_consumed_id_as_a_parent() { + // Fusion (many:1), the sole escape hatch and the only place any id is + // named at record time. The product's lineage is the product of every + // origin's, which is what the parents column carries. + let (slot, fused) = (NodeId::fresh(), NodeId::fresh()); + let mut product = NodeId::PLACEHOLDER; + let table = recorded(|| { + let g = enter(slot, "rw.fuse", Nature::Machinery); + g.also_consumes(fused); + product = Expr::lit(Lit::Int(1)).node_id(); + }); + assert_eq!( + table.parents(product), + &[slot, fused], + "a fusion's whole consumed set is the product's parent set", + ); } #[test] - fn end_to_end_session_step_feeds_collapse() { - // Ties (a) and (b): record a real rewrite through a session, then fold - // the drained log with `collapse` and check the resulting relation. + fn a_flush_carries_the_blame_channel_into_the_row_unmerged() { + let (slot, blamed) = (NodeId::fresh(), NodeId::fresh()); + let mut product = NodeId::PLACEHOLDER; + let table = recorded(|| { + let g = enter(slot, "rw.blamed", Nature::Machinery); + g.blame(&[blamed]); + product = Expr::lit(Lit::Int(1)).node_id(); + }); + assert_eq!(table.parents(product), &[slot], "blame is not a parent"); + assert_eq!(table.blame(product), &[blamed], "and a parent is not blame"); + } + + #[test] + fn a_recording_over_an_untouched_node_records_nothing() { + // A rule that inspects and declines. No mint, no copy, id unchanged: + // the *preserve*, and it must not cost a row — this is what lets a pass + // open a recording on every rewrite *attempt* rather than every firing. + let e = Expr::lit(Lit::Int(1)); + let table = recorded(|| { + let _g = enter(e.node_id(), "rw.noop", Nature::Machinery); + }); + assert_eq!(table.len(), 0); + } + + #[test] + fn a_recording_over_an_in_place_mutation_records_nothing() { + // `simplify`'s `*op = BinOpKind::Concat` shape: the node's *value* + // changes, its identity does not. A preserve, and correctly silent — + // the node's lineage is the self-edge it already had. + let mut e = Expr::lit(Lit::Int(1)); + let id = e.node_id(); + let table = recorded(|| { + let _g = enter(id, "rw.in_place", Nature::Machinery); + e.node = TypedExprNode::Lit(Lit::Int(2)); + }); + assert_eq!(e.node_id(), id); + assert_eq!(table.len(), 0); + } + + // ---- a recording declares nothing -------------------------------------- + // + // The property under test throughout: a site names the node being rewritten, + // births are captured, and fate is the relation's live-set difference. These + // are the pass/fail statements behind the design. + + #[test] + fn an_end_to_end_recording_feeds_the_fold() { + // Record a real rewrite, then fold the rows and check + // the resulting relation and attribution. let a = Expr::lit(Lit::Int(1)); let a_id = a.node_id(); - - let session = RecorderSession::new(); - let out_id; - { - let _g = step("rw.replace", vec![a_id], vec![a_id], Nature::Expansion); + let mut out_id = NodeId::PLACEHOLDER; + let table = recorded(|| { + let _g = enter(a_id, "rw.replace", Nature::Expansion); out_id = Expr::lit(Lit::Int(2)).node_id(); - } - let log = session.into_log(); + }); - let logs = vec![(Pass::Inline, log)]; let (map, proj, leaks) = collapse( - &logs, + &table, + PASSES, &set([a_id]), &set([out_id]), &SourceProjection::new(), ); - assert!(leaks.is_empty(), "{leaks:?}"); - assert_eq!(map.upstream(&out_id), &[a_id]); - assert_eq!(map.downstream(&a_id), &[out_id]); - // The output attributes to the (blamed) input via this pass. + // `a` is replaced, so it dies — reported, not a defect. + assert!(defects(&leaks).is_empty(), "{leaks:?}"); + assert_eq!(ids_of(map.upstream(&out_id)), vec![a_id]); + assert_eq!(ids_of(map.downstream(&a_id)), vec![out_id]); let attr = proj.get(&out_id).expect("output attributed"); - let tag = &attr.rewritten; - assert_eq!(tag.via, Pass::Inline); - assert_eq!(tag.label, "rw.replace"); + assert_eq!(attr.rewritten.via, Pass::Inline); + assert_eq!(attr.rewritten.label, "rw.replace"); } #[test] fn born_copied_discarded_template_composes_without_leaks() { // The `fold_induction_loop`/`build_writer` template shape (transact/letrec - // instrumentation hazard): a single frame births a template `T`, copies - // it per read site (via `Subst::discharge_env_in_place`), and discards - // `T` (it never reaches the output tree). Per `OpenStep::flush_into`'s - // refinement the frame's `Transform` (which produces `T`) flushes BEFORE - // the captured `Copy` steps, so `T` is live when the copies are processed — - // the whole shape composes in ONE frame, no split needed. `T` remains live - // at collapse (never consumed) but is neither an input-pane nor an - // output-pane id, so it triggers no leak (`Dropped` checks inputs only, - // `Unexplained` checks outputs only). - let origin = NodeId::fresh(); // an input-pane id the template descends from - let (t, c1, c2); - let session = RecorderSession::new(); - { - let _g = step( - "test.template", - vec![origin], - vec![origin], - Nature::Machinery, - ); - // Birth the template inside the frame (a mint captured as `produced`). + // instrumentation hazard): one recording births a template `T`, copies + // it per read site (the read-your-writes environment discharge freshens + // a clone's interior at each), and discards `T` (it never reaches the + // output tree). `T` is neither an input-pane nor an output-pane id, so + // it triggers no leak (`Died` checks inputs only, `Unexplained` checks + // outputs only) — the whole shape composes in ONE recording, no split + // needed. + let origin = NodeId::fresh(); // the named slot, an input-pane id + let (mut t, mut c1, mut c2) = ( + NodeId::PLACEHOLDER, + NodeId::PLACEHOLDER, + NodeId::PLACEHOLDER, + ); + let table = recorded(|| { + let _g = enter(origin, "test.template", Nature::Machinery); + // Birth the template inside the recording (a mint captured as a + // birth). let template = Expr::lit(Lit::Int(0)); t = template.node_id(); // Copy it twice — one freshened clone per read site. @@ -2197,41 +3314,252 @@ mod tests { c1 = r1.node_id(); let r2 = template.clone(); c2 = r2.node_id(); - } - let log = session.into_log(); - // Flush order: the Transform producing T first, then a Copy per origin. + }); + assert_eq!(table.parents(t), &[origin]); + assert_eq!(table.parents(c1), &[t], "the copies row on the template"); + assert_eq!(table.parents(c2), &[t]); + + // The sweep's premise on rows written by the *recorder* rather than by + // hand: capture-only births plus one monotone counter means no edge runs + // backwards, so no vertex is ever revisited. assert_eq!( - log[0].op, - Op::Transform { - consumed: vec![origin], - produced: vec![t], - }, - "the Transform (producing T) flushes before the captured copies: {log:?}", - ); - assert!( - log[1..] - .iter() - .all(|s| matches!(&s.op, Op::Copy { origin: o, .. } if *o == t)), - "the per-origin copies of T flush after it is live: {log:?}", + sweep_metrics(&table, PASSES, &set([origin])).backward_edges, + 0, + "a captured record's edges all run from smaller NodeId to larger", ); - - let logs = vec![(Pass::Inline, log)]; let (map, _proj, leaks) = collapse( - &logs, + &table, + PASSES, &set([origin]), &set([c1, c2]), &SourceProjection::new(), ); assert!( - leaks.is_empty(), - "born-copied-discarded template composes leak-free in one frame: {leaks:?}", + defects(&leaks).is_empty(), + "born-copied-discarded template composes defect-free in one recording: {leaks:?}", ); - // c1/c2 carry T's roots — the frame's consumed lineage (`origin`). - assert_eq!(map.upstream(&c1), &[origin]); - assert_eq!(map.upstream(&c2), &[origin]); + // c1/c2 carry T's roots — the recording's parentage (`origin`). + assert_eq!(ids_of(map.upstream(&c1)), vec![origin]); + assert_eq!(ids_of(map.upstream(&c2)), vec![origin]); assert_eq!( - sorted(map.downstream(&origin).to_vec()), + sorted(ids_of(map.downstream(&origin))), sorted(vec![c1, c2]) ); } + + #[test] + fn a_recorded_wrap_does_not_claim_the_wrapped_node_died() { + // The adopt-a-live-subtree shape: mint a wrapper *over* the named node, + // which stays in the tree as a child. Both ids are live at the + // fold; it must report no death and no leak. A record that + // declared the slot consumed would report it dead. + let slot = NodeId::fresh(); + let mut wrapper = NodeId::PLACEHOLDER; + let table = recorded(|| { + let _g = enter(slot, "rw.wrap", Nature::Machinery); + wrapper = Expr::lit(Lit::Int(0)).node_id(); + }); + let (map, _proj, leaks) = collapse( + &table, + PASSES, + &set([slot]), + &set([slot, wrapper]), + &SourceProjection::new(), + ); + assert!(leaks.is_empty(), "{leaks:?}"); + assert_eq!(ids_of(map.upstream(&wrapper)), vec![slot]); + assert_eq!( + ids_of(map.upstream(&slot)), + vec![slot], + "the wrapped node survives" + ); + } + + #[test] + fn deaths_are_the_live_set_difference_not_a_declaration() { + // The fate-prediction replacement, end to end. One rewrite; whether the + // named node survives is decided *only* by which snapshot it is in. + // The identical record yields "survived" against one output pane and + // "died" against another, and no site said either. + let make = || { + let slot = NodeId::fresh(); + let mut born = NodeId::PLACEHOLDER; + let table = recorded(|| { + let _g = enter(slot, "rw.maybe_drop", Nature::Expansion); + born = Expr::lit(Lit::Int(1)).node_id(); + }); + (slot, born, table) + }; + + let (slot, born, table) = make(); + let (_m, _p, leaks) = collapse( + &table, + PASSES, + &set([slot]), + &set([slot, born]), + &SourceProjection::new(), + ); + assert!(leaks.is_empty(), "survivor pane: {leaks:?}"); + + let (slot, born, table) = make(); + let (map, _p, leaks) = collapse( + &table, + PASSES, + &set([slot]), + &set([born]), + &SourceProjection::new(), + ); + assert_eq!( + leaks, + vec![Leak::Died { input: slot }], + "nothing declares a fate, so `Died` IS the death report", + ); + assert_eq!( + ids_of(map.upstream(&born)), + vec![slot], + "lineage survives the death" + ); + } + + // ---- the node table's columns ------------------------------------------ + + fn tag(label: RewriteLabel) -> RewriteTag { + RewriteTag { + via: TEST_PASS, + nature: Nature::Machinery, + label, + } + } + + #[test] + fn a_row_round_trips_every_column() { + let mut table = LineageTable::default(); + let [node, parent, blamed] = ids(); + let rule = table.intern_rule(tag("rw.one")); + table.record(node, &[parent], &[blamed], rule); + + assert_eq!(table.parents(node), &[parent]); + assert_eq!(table.blame(node), &[blamed]); + assert_eq!(table.rule(node), Some(tag("rw.one"))); + assert!(table.contains(node)); + } + + #[test] + fn a_fusion_row_round_trips_all_its_parents() { + // The many:1 shape: one product, every consumed id a parent. Parents are + // a set of edges, not a single channel, so none of them may be dropped + // or reordered into a "primary". + let mut table = LineageTable::default(); + let [node, p0, p1, p2] = ids(); + let rule = table.intern_rule(tag("rw.fuse")); + table.record(node, &[p0, p1, p2], &[], rule); + + assert_eq!(table.parents(node), &[p0, p1, p2]); + assert!( + table.blame(node).is_empty(), + "no blame is not empty parents" + ); + } + + #[test] + fn an_unrecorded_id_reads_empty_rather_than_panicking() { + // The predicate-interior case: a real id from the same counter that no + // recording ever produced. Every read must have an answer for it. + let mut table = LineageTable::default(); + let [parent, recorded, never] = ids(); + let rule = table.intern_rule(tag("rw.one")); + table.record(recorded, &[parent], &[], rule); + + assert!(table.parents(never).is_empty()); + assert!(table.blame(never).is_empty()); + assert_eq!(table.rule(never), None); + assert_eq!(table.rule_in(never, PASSES), None); + assert!(!table.contains(never)); + } + + #[test] + fn a_row_outside_the_passes_reads_as_unrecorded_to_that_relation() { + let mut table = LineageTable::default(); + let [parent, node] = ids(); + let rule = table.intern_rule(RewriteTag { + via: Pass::Mono, + ..tag("rw.one") + }); + table.record(node, &[parent], &[], rule); + + assert!(table.rule(node).is_some(), "the row exists"); + assert_eq!( + table.rule_in(node, PASSES), + None, + "but not to a pane relation whose passes exclude it", + ); + } + + #[test] + fn deaths_never_name_an_id_no_row_recorded() { + // The single most important invariant: the death set is + // `recorded ∖ live`, taken over rows and never over the key space. The + // key space is the global counter, so it addresses ids this compile never + // built at all; a difference taken over it would report every one of them + // as a death. Predicate interiors used to be the standing example — they + // were addressed but unrecorded, and absent from the live set too — and + // are no longer, since `collect_tree_ids` enumerates them and the fold + // records them. They now cancel from both sides, which is why admitting + // them did not move the death counts. + let mut table = LineageTable::default(); + let [parent, survivor, dead, never] = ids(); + let rule = table.intern_rule(tag("rw.one")); + table.record(survivor, &[parent], &[], rule); + table.record(dead, &[parent], &[], rule); + + let live: HashSet = set([survivor]); + let deaths = table.deaths(&live); + assert_eq!( + deaths, + vec![dead], + "only a recorded id absent from the live set is a death", + ); + assert!( + !deaths.contains(&never), + "an id with no row describes no node that could have died", + ); + } + + #[test] + fn equal_tags_share_one_rule_id_and_distinct_tags_do_not() { + let mut table = LineageTable::default(); + let [parent, a, b, c] = ids(); + let one = table.intern_rule(tag("rw.one")); + let one_again = table.intern_rule(tag("rw.one")); + let other = table.intern_rule(tag("rw.other")); + assert_eq!(one, one_again, "the tag is the interning key, by value"); + assert_ne!(one, other); + + table.record(a, &[parent], &[], one); + table.record(b, &[parent], &[], one_again); + table.record(c, &[parent], &[], other); + assert_eq!( + table.rule_id(a), + table.rule_id(b), + "two rows naming one rewrite hold one handle", + ); + assert_ne!(table.rule_id(a), table.rule_id(c)); + assert_eq!(table.rule_count(), 2, "two distinct tags, two entries"); + assert_eq!(table.rule(c), Some(tag("rw.other"))); + } + + #[test] + fn a_differing_nature_is_a_distinct_rule_despite_a_shared_label() { + // The triple is interned whole: one arm of a rewrite relabelled to a + // different nature is a different rule, which is exactly why a site that + // needs two natures opens a second recording rather than mutating one. + let mut table = LineageTable::default(); + let machinery = table.intern_rule(tag("rw.one")); + let expansion = table.intern_rule(RewriteTag { + nature: Nature::Expansion, + ..tag("rw.one") + }); + assert_ne!(machinery, expansion); + assert_eq!(table.rule_count(), 2); + } } diff --git a/src/ccl/lower/comprehension.rs b/src/ccl/lower/comprehension.rs index 9b866813..ea99d942 100644 --- a/src/ccl/lower/comprehension.rs +++ b/src/ccl/lower/comprehension.rs @@ -342,8 +342,11 @@ pub(super) fn lower_list_comp( /// Hand out a tree copy of `origin` for one arm of a fan-out. Every arm is a /// sibling, including the first: a fan-out places the same subtree under several -/// arms and no arm is privileged. The copy-frame records each copy as a `Copy` of +/// arms and no arm is privileged. The copy sink records each copy as a `Copy` of /// the origin, so every arm's attribution mirrors the original's. +/// +/// Keeping the first arm's ids was measured at 30 ids saved over the whole +/// pipeline suite, max subtree 5 — which does not pay for a second code path. fn fan_out_copy(origin: &Expr, label: &'static str) -> Expr { use crate::ccl::lineage::copy_frame; let _frame = copy_frame(label); @@ -434,6 +437,8 @@ fn fan_out_element_case( // `true → Case{…}`) into one flat partition, so each arm is a plain value. let branches = flatten_trailing_value_case(branches); let mut prior_guards: Vec = Vec::new(); + // The source subtree is placed once per arm in the element map and once more in + // that arm's gate, so every use after the first must be a freshened copy. let arms: Vec = branches .into_iter() .map(|b| { @@ -467,10 +472,10 @@ fn fan_out_element_case( ), Expr::lambda(iter_var, Type::Hole, gate), ); - // `gate_on_source` rides the cast target's refinement predicate, so - // its interior is in the domain the fold must explain and nothing in - // the main-tree walk reaches it. Sweep it - // (`src/ccl/design/provenance.md`, "Walking the ids"). + // `gate_on_source` rides the cast target's refinement predicate — a + // type slot outside the `walk_children` walk — so nothing else will + // record its interior, and `collect_tree_ids` now enumerates it. + // Sweep it (`design/provenance.md`, "Walking the ids", crossing 1). ctx.tag_predicate(&gate_on_source, span, "lower.comp_arm_gate_pred"); let target = refined_data_fun(Type::Hole, gate_on_source, Type::Hole); ctx.tag_machinery(make_cast(elem_map, target), span, ec) diff --git a/src/ccl/lower/exprs.rs b/src/ccl/lower/exprs.rs index 2dd53fee..4d1889a3 100644 --- a/src/ccl/lower/exprs.rs +++ b/src/ccl/lower/exprs.rs @@ -464,7 +464,7 @@ pub(super) fn lower_compare( // Build one BinOp per (op, adjacent-operand-pair). Each middle operand is // placed in two pairs, and no placement is privileged, so every placement is a - // freshened copy taken inside a lowering copy-frame: each re-minted node lands + // freshened copy taken inside a lowering copy sink: each re-minted node lands // as a `Copy` step mirroring the original operand's (Source) image, which is // the attribution wanted for a duplicated operand. let operand = |i: usize| { @@ -482,6 +482,10 @@ pub(super) fn lower_compare( CmpOp::Gt => CompareKind::Greater, CmpOp::GtE => CompareKind::GreaterOrEq, }; + // Operand `i` is this pair's left side and, for `i > 0`, was pair `i-1`'s + // right side; operand `i+1` is this pair's right side and may be the next + // pair's left. Every placement freshens and is recorded, so which use + // comes first does not matter here. let lhs = operand(i); let rhs = operand(i + 1); // Each pair comparison images its `` in the chain, spanning its two @@ -617,19 +621,19 @@ mod tests { /// Regression: a chained comparison shares each middle operand between two /// adjacent pairs. A bare clone would put the same `NodeId`s in the tree /// twice, tripping `assert_unique_node_ids` at the `"post-lowering"` - /// boundary. The second use is freshened inside a lowering copy-frame; the tree must be + /// boundary. The second use is freshened inside a lowering copy sink; the tree must be /// duplicate-free, and the lowering fold must explain every node with no leak /// (the freshened copy resolves as a `Copy` mirroring its origin's image). #[test] fn chained_compare_freshens_shared_operands() { use crate::ccl::context::{assert_unique_node_ids, collect_tree_ids}; - use crate::ccl::lineage::{RecorderSession, collapse_lowering}; + use crate::ccl::lineage::{LoweringSession, collapse_lowering}; let expr = parse_expr("1 < x < 3"); let mut ctx = LoweringContext::default(); - let session = RecorderSession::lowering(); + let session = LoweringSession::install(); let ccl = lower_expr(&expr, &mut ctx).expect("lowering failed"); - let log = session.into_lowering_log(); + let log = session.into_log(); // The same tripwire the pipeline runs at every pass boundary — this test // is the crafted program for the class it guards. diff --git a/src/ccl/lower/functions.rs b/src/ccl/lower/functions.rs index 910ff2a2..32b049e8 100644 --- a/src/ccl/lower/functions.rs +++ b/src/ccl/lower/functions.rs @@ -200,7 +200,7 @@ pub(super) fn uncurry_params( // three nodes as machinery leaves. // // The template itself never enters the tree. Its two interior ids reach - // the tree as the freshened copies the frame below captures, so they are + // the tree as the freshened copies the recording below captures, so they are // origins of live nodes; its ROOT does not — each occurrence's own id // replaces it — so that one id is tagged and then carried by nothing. // Harmless in the product (the projection is filtered to the output tree, @@ -280,7 +280,7 @@ pub(super) fn lower_lambda( /// what buys occurrence fidelity here: every `x` in `def f(x, y)`'s body becomes /// its own `__arg_tuple.0` whose root still points at that `x`'s span, rather /// than every projection mirroring the one shared `def` span. The interior -/// freshens land as ambient `Copy`s in the open lowering copy-frame below, +/// freshens land as ambient `Copy`s in the open lowering copy sink below, /// mirroring the template's machinery attribution — so every substituted node /// stays covered by the lowering fold. /// @@ -292,8 +292,8 @@ pub(super) fn lower_lambda( fn substitute_param_in_body(expr: Expr, name: &Name, replacement: &Expr) -> Expr { use crate::ccl::lineage::copy_frame; let mut expr = expr; - // A lowering copy-frame: the discharge's interior freshens fire `on_copy` - // into this frame, which flushes them as `Copy` LoweringSteps (mirroring the + // A lowering copy sink: the discharge's interior freshens fire `on_copy` into + // this recording, which writes them as `Copy` LoweringSteps (mirroring the // template) into the always-on lowering log. A no-op when no session is // installed (the lower unit tests). let _frame = copy_frame("lower.uncurry_proj"); @@ -443,7 +443,7 @@ in add" #[test] fn uncurry_projection_roots_carry_occurrence_spans() { use crate::ccl::TypedExprNode; - use crate::ccl::lineage::{Nature, RecorderSession, SourceProjection, collapse_lowering}; + use crate::ccl::lineage::{LoweringSession, Nature, SourceProjection, collapse_lowering}; use crate::ccl::provenance::NodeId; use crate::chl_parser::ast::Span; use std::collections::HashSet; @@ -457,11 +457,11 @@ in add" let stmts = parse_module(src); let mut ctx = LoweringContext::default(); - let session = RecorderSession::lowering(); + let session = LoweringSession::install(); let ccl = lower_stmts(&stmts, &mut ctx) .into_result() .expect("lowering failed"); - let log = session.into_lowering_log(); + let log = session.into_log(); let mut ids: HashSet = HashSet::new(); fn all_ids(e: &Expr, acc: &mut HashSet) { diff --git a/src/ccl/lower/loops.rs b/src/ccl/lower/loops.rs index 04b77b65..7c3a16ff 100644 --- a/src/ccl/lower/loops.rs +++ b/src/ccl/lower/loops.rs @@ -41,7 +41,7 @@ fn stmt_has_yield(stmt: &Spanned) -> bool { /// (checked recursively through `if` guards and nested `for` loops), mirroring /// [`for_body_has_yield`]. /// -/// Used to keep a feed-bearing side-effect loop on the `Compose`/`desugar` +/// Used to keep a feed-bearing side-effect loop on the `Compose`/`channelize` /// path: only a loop with neither a feed nor a yield (a bare-effect body such /// as `for x: bump(cnt)`, whose only possible effect is a hidden mutable write /// inside a call) is routed to the direct-mirror `For` marker. @@ -738,7 +738,7 @@ pub(super) fn find_nested_mutation_var( /// `bump(cnt)`) becomes a side-effect `ExprStmt`, and their embedded reads /// stay bare `Var`s. The phase threads the recurrence, the read-your-writes /// shadowing, and hoists each in-loop feed to an ordinary feed of the loop's -/// history for desugar to route. Other assignments are per-iteration `Let`s. +/// history for channelize to route. Other assignments are per-iteration `Let`s. /// /// `acc_names` may be empty: a bare-effect loop (`for x: bump(cnt)`) has no /// *visible* accumulator, since the write is hidden inside a call and only diff --git a/src/ccl/lower/stmts.rs b/src/ccl/lower/stmts.rs index 57f07394..60a7b0af 100644 --- a/src/ccl/lower/stmts.rs +++ b/src/ccl/lower/stmts.rs @@ -749,7 +749,7 @@ pub(super) fn lower_middle_stmt( } // A feed/yield loop with no accumulator is a side-effecting - // `Compose` (desugar routes its feeds). + // `Compose` (channelize routes its feeds). let for_expr = lower_generator_for(target, iter, for_body, &scope, stmt.span, ctx)?; Ok(ctx.tag_machinery(Expr::expr_stmt(for_expr, body), stmt.span, "lower.stmt_seq")) } @@ -1924,7 +1924,7 @@ x"; #[test] fn lowering_tags_nodes_with_source_spans() { use crate::ccl::TypedExprNode; - use crate::ccl::lineage::{Nature, RecorderSession, collapse_lowering}; + use crate::ccl::lineage::{LoweringSession, Nature, collapse_lowering}; use crate::ccl::provenance::NodeId; use crate::chl_parser::ast::Span; use std::collections::HashSet; @@ -1941,11 +1941,11 @@ x"; // Install the always-on lowering session, lower, then fold the log into // the lowering projection — the same handoff `compile_program` runs. let mut ctx = LoweringContext::default(); - let session = RecorderSession::lowering(); + let session = LoweringSession::install(); let lowered = lower_stmts(&stmts, &mut ctx) .into_result() .expect("lowering succeeds"); - let log = session.into_lowering_log(); + let log = session.into_log(); let mut output_ids: HashSet = HashSet::new(); fn ids(e: &Expr, acc: &mut HashSet) { acc.insert(e.node_id()); diff --git a/src/ccl/mut_elim.rs b/src/ccl/mut_elim.rs index e6b644db..73e3235f 100644 --- a/src/ccl/mut_elim.rs +++ b/src/ccl/mut_elim.rs @@ -50,6 +50,7 @@ use crate::ccl::{ TypedExprNode, ccl_utils::{COMMIT_SELECTOR, strip_refinements, synthesize_arm_predicate, typed_compose}, letrec::check_letrec_causal, + lineage, subst::Subst, symbolic::symbolic, }; @@ -290,7 +291,7 @@ fn hoist_writer_body(binding: TypedBinding, writer_body: Expr, body: Expr) -> Ex /// /// The `Let`-hoist is gated on `spine_writes_mut`: only a genuine writer body /// is reassociated. `Feed`/`Define`-headed `ExprStmt` chains keep their nesting -/// — desugar collects feeds outermost-first, so reassociating them would +/// — channelize collects feeds outermost-first, so reassociating them would /// reorder channel contributions — and a pure `Let` (e.g. a join subplan) holds /// no `MutWrite` on its spine, so it is left undisturbed. After this pass the /// only `MutWrite`s in the tree are `ExprStmt` effects, so `rewrite` and @@ -356,6 +357,7 @@ fn flatten_spine(mut e: Expr) -> Expr { if let TypedExprNode::Let { bound_expr, .. } = &e.node && spine_writes_mut(bound_expr) { + let let_id = e.node_id(); let TypedExprNode::Let { binding, bound_expr, @@ -364,12 +366,42 @@ fn flatten_spine(mut e: Expr) -> Expr { else { unreachable!() }; - return flatten_spine(hoist_writer_body(binding, *bound_expr, *body)); + // Unlike the two reassociations above, this one is not 1:1: the hoist + // splices `let y = ⟨terminal⟩` into the body's terminal position and + // wraps the lifted write in a statement, so it *mints* — the `ExprStmt`, + // the spliced `let`, and its `unit` value. They stand in for the `Let` + // being hoisted, so that is the slot. `Machinery`, because the spliced + // binding is plumbing that restores the flat-spine invariant rather than + // anything the user wrote. + // + // The recursion is outside the recording, so a nested hoist attributes to its + // own `Let`. + let hoisted = { + let _g = lineage::enter( + let_id, + "letrec.hoist_writer_body", + lineage::Nature::Machinery, + ); + hoist_writer_body(binding, *bound_expr, *body) + }; + return flatten_spine(hoisted); } // A bare write reached in value/terminal position: it is a `Unit`-valued // statement, not a value to bind. if is_mut_write(&e) { - return flatten_spine(Expr::expr_stmt(e, unit_expr())); + // The write keeps its own id and becomes the effect; the `ExprStmt` and + // the `unit` body are new, and they exist to put this write in statement + // position. So the write is the slot. + let write_id = e.node_id(); + let terminalized = { + let _g = lineage::enter( + write_id, + "letrec.terminalize_write", + lineage::Nature::Machinery, + ); + Expr::expr_stmt(e, unit_expr()) + }; + return flatten_spine(terminalized); } // Pass-through — recurse in place so this node's own `ty`/annotation are // preserved (rebuilding would drop them, corrupting e.g. join subplans). @@ -387,13 +419,28 @@ fn flatten_spine(mut e: Expr) -> Expr { } fn rewrite(mut expr: Expr) -> Expr { + let stmt_id = expr.node_id(); if let TypedExprNode::ExprStmt { expr: effect, body } = expr.node { + let effect_id = effect.node_id(); if let TypedExprNode::For { target, iter, body: loop_body, } = effect.node { + // The statement is the slot: the causal `LetRec` replaces it. + // + // There is deliberately no drop-path test. Whether the whole loop + // vanishes — no accumulator, no feed, e.g. a transaction-emptied + // `For` — is read off the live-set difference, so this site does not + // predict it. Predicting it meant re-running `collect_writes` and + // `body_has_feed` here to guess what `transform_loop` would decide + // ~140 lines away. + // + // `blame` names the `For` rather than the `ExprStmt` so the products + // resolve to the loop keyword's span, not the statement's. + let g = lineage::enter(stmt_id, "letrec.loop", lineage::Nature::Expansion); + g.blame(&[effect_id]); return transform_loop(target, *iter, *loop_body, *body); } // A `MutWrite` outside any `For` is a *sequential* mutation — a @@ -401,6 +448,12 @@ fn rewrite(mut expr: Expr) -> Expr { // (`bump(cnt)`) spliced between statements. There is no recurrence to // build; normalize it to a shadowing `let` (see `normalize_bare_write`). if let TypedExprNode::MutWrite { name, value } = effect.node { + // The shadowing `let` this mints is captured against the statement + // node it replaces. The `MutWrite` marker and the `ExprStmt` wrapper + // both vanish, but neither is named: both are absent from the output + // tree, so the boundary difference reports them. + let g = lineage::enter(stmt_id, "letrec.bare_write", lineage::Nature::Machinery); + g.blame(&[effect_id]); return normalize_bare_write(name, *value, *body); } // Not a loop/write statement: rebuild and recurse. @@ -503,7 +556,7 @@ pub(crate) fn fun_parts(ty: &Type) -> (Type, Type) { /// fresh record field carrying its per-iteration value, and that value /// (already resolved in the read-your-writes environment at the feed site). /// The loop's history binding computes the field alongside the recurrence; -/// the phase hoists `Feed(defer, __hist ▷ .field)` out of the loop so desugar +/// the phase hoists `Feed(defer, __hist ▷ .field)` out of the loop so channelize /// routes it as an ordinary channel contribution. struct FeedSite { defer: Name, @@ -998,7 +1051,7 @@ fn transform_feed_only_loop(target: TypedBinding, iter: Expr, loop_body: Expr, c `with begin():` block, so a `For` here always carries a feed" ); let mut body_out = rewrite(cont); - // Emit in reverse so the first source feed ends up outermost — desugar + // Emit in reverse so the first source feed ends up outermost — channelize // collects feeds outermost-first into the channel union, preserving source // order (mirrors the accumulator path's hoist ordering). for (defer, value) in feeds.into_iter().rev() { @@ -1593,9 +1646,20 @@ fn attach_feed_fields(decision: Expr, feeds: &[FeedSite]) -> Expr { } => { let new_body = attach_feed_fields(*body, feeds); let ty = new_body.ty.clone(); - // The same logical `Let` with its feed fields attached, so it keeps its - // own id rather than minting a replacement. - let mut e = Expr::let_in_preserving(node_id, binding, *bound_expr, new_body); + // The same logical `Let` with its feed fields attached, so it keeps + // its own id rather than minting a replacement. `preserve` builds it + // at that id directly; `Expr::let_in(..).re_root(id)` used to mint one + // and overwrite it, which fires `on_mint` for an id no node ends up + // carrying — the phantom birth `preserve` exists to make + // unrepresentable. + let mut e = Expr::preserve( + node_id, + TypedExprNode::Let { + binding, + bound_expr, + body: Box::new(new_body), + }, + ); e.ty = ty; e } @@ -1827,7 +1891,7 @@ mod tests { /// /// Unit-test form at the letrec boundary (the plan's RT-4b fallback): the /// bare-writer `MutWrite` is consumed by the loop rewrite and does not survive - /// into `post_desugar_ir` as a span-indexable `MutWrite`, so id preservation + /// into `post_channelize_ir` as a span-indexable `MutWrite`, so id preservation /// through the phase is asserted directly here. #[test] fn flatten_spine_bare_writer_preserves_id() { diff --git a/src/ccl/names.rs b/src/ccl/names.rs index 23777b12..db4331c4 100644 --- a/src/ccl/names.rs +++ b/src/ccl/names.rs @@ -98,7 +98,7 @@ pub enum SyntheticKind { /// `Name` may itself be a `Synthetic` carrying a `SyntheticKind`, so the /// reference must be indirected to keep the type finite-sized.) Mono(Box), - /// A lambda/binding floated out during defer desugaring. + /// A lambda/binding floated out during channelization. FloatedDefer, /// The fresh binder the solver mints for a dependent application's /// expected Pi type (`(__arg: d) ⇒ result`), discharged to the argument. @@ -191,7 +191,7 @@ impl Name { Self::synthetic(SyntheticKind::Mono(Box::new(source))) } - /// A lambda/binding floated out during defer desugaring. + /// A lambda/binding floated out during channelization. pub fn floated() -> Self { Self::synthetic(SyntheticKind::FloatedDefer) } diff --git a/src/ccl/planning/groupby.rs b/src/ccl/planning/groupby.rs index c2a5c46a..eb61d7d9 100644 --- a/src/ccl/planning/groupby.rs +++ b/src/ccl/planning/groupby.rs @@ -273,7 +273,7 @@ mod tests { /// tree, and the lifted copy must not carry ids that are still live /// elsewhere. Today `lambda_elim::run` provides that by rebuilding the term; /// this pins the *property* rather than the mechanism, so an elim that - /// started preserving ids fails here instead of at a pane boundary. + /// started preserving ids fails here instead of at a pane relation. #[test] fn groupby_recognition_lifts_the_key_without_aliasing() { let key = var("key").with_ty(fun_ty(int_ty(), int_ty())); diff --git a/src/ccl/planning/iterate.rs b/src/ccl/planning/iterate.rs index d7516365..413d09bf 100644 --- a/src/ccl/planning/iterate.rs +++ b/src/ccl/planning/iterate.rs @@ -9,6 +9,7 @@ use std::mem::take; use super::join::try_hash_join_rewrite; use crate::ccl::ccl_utils::PredMemo; +use crate::ccl::lineage; use super::predicates::{compile_refinement_predicates, fn_of_bare_predicate}; use super::*; @@ -393,6 +394,17 @@ pub(super) fn wrap_with_iterate(expr: &mut Expr) { let Some(domain_ty) = expr.ty.domain() else { return; }; + // The site being wrapped is the slot. The predicate `fresh_copy` below lands + // as a `Copy` of the term it lifts out of the type. + // + // These rows reach no table in a normal compile: `compile_program` calls + // `planning::run` outside every pass scope it opens, so they land only under + // `CAMBRA_LINEAGE_AUDIT=planning` (`recognized..join-planned`). + let _g = lineage::enter( + expr.node_id(), + "planning.iterate", + lineage::Nature::Machinery, + ); // Walk every nested `Type::Refinement` layer (innermost ⊇ outermost, // each layer's predicate must hold), collecting the predicates // outer-to-inner; reverse to inner-to-outer. Then emit a uniform @@ -407,10 +419,16 @@ pub(super) fn wrap_with_iterate(expr: &mut Expr) { let mut preds: Vec = Vec::new(); let mut current = &domain_ty; while let Type::Refinement(base, refinement) = current { - // Lifting a predicate out of a *type* and into the term tree: a predicate - // interior may already alias a live main-tree id, and one predicate `Rc` - // reached from two iteration sites would land twice. - preds.push(fn_of_bare_predicate(base.as_ref(), &refinement.predicate).clone()); + // Lifting a predicate out of a *type* and into the term tree: one + // predicate `Rc` reached from two iteration sites would otherwise land + // twice. `fn_of_bare_predicate` already returns an owned, freshened term + // on both its paths — the fast path clones the function subterm, the slow + // path η-expands and runs `lambda_elim` — so the lift is already a + // distinct node-set and needs no second copy here. (It needed one when + // `Clone` preserved ids; the trailing `fresh_copy()` was load-bearing + // then, and cloning again now would duplicate a whole tree per + // refinement layer for nothing.) + preds.push(fn_of_bare_predicate(base.as_ref(), &refinement.predicate)); current = base.as_ref(); } preds.reverse(); @@ -447,9 +465,14 @@ pub(super) fn wrap_with_iterate(expr: &mut Expr) { }; let mut elts: Vec = vec![source]; match body.node { - TypedExprNode::Compose(existing) => { - elts.extend(existing); - } + // The body's own `Compose` is dissolved and its elements spliced into + // the new chain. Those elements keep their ids and become children, so + // the only node that vanishes is the `Compose` itself — which is the + // node this recording named (`body` is `expr`, taken above). Its fate is + // the boundary's live-set difference, so there is nothing to declare: + // `FrameGuard::also_consumes` exists for a node the construction hooks + // *cannot* see, and this one is the named slot. + TypedExprNode::Compose(existing) => elts.extend(existing), _ => elts.push(body), } // The override is now redundant — `source`'s domain already carries diff --git a/src/ccl/provenance.rs b/src/ccl/provenance.rs index 2109560f..4940ccc2 100644 --- a/src/ccl/provenance.rs +++ b/src/ccl/provenance.rs @@ -2,12 +2,16 @@ //! //! # Purpose //! -//! Cambra lowers source through a chain of passes (lowering → uniquify → -//! infer → inline → lambda-elim → planning). Every IR expression node carries a -//! stable [`NodeId`] so its identity survives across those passes, and each -//! rewrite records — through the lineage recorder in [`crate::ccl::lineage`] — -//! how the node it produced relates to the nodes it consumed. That lineage -//! folds, at each inspector pane boundary, into a +//! Cambra lowers source through a chain of passes, and [`Pass`] tags the one +//! that produced a node. In pipeline order those are `Lower`, `Uniquify`, +//! `Mono` (inside inference), `Inline`, `Transact`, `Letrec`, `Channelize`, +//! `LambdaElim`, `Planning` — the order `compile_program` runs them in +//! (`crate::ccl::context`). Every IR expression node carries a stable +//! [`NodeId`] so its identity survives across those passes, and each rewrite +//! records — through the lineage recorder in [`crate::ccl::lineage`] — how the +//! node it produced relates to the nodes it consumed. +//! `CompiledProgram::materialize_panes` folds that lineage, at each pane +//! boundary, into a //! [`SourceProjection`](crate::ccl::lineage::SourceProjection) mapping a node //! back to the source spans it traces to. This module owns only the two //! identity primitives that lineage is parameterized over: the node id and the @@ -63,16 +67,6 @@ impl NodeId { pub fn fresh() -> Self { NodeId(FRESH_NODE_ID.fetch_add(1, Ordering::Relaxed)) } - - /// The id's underlying number, for use as an opaque serialization handle - /// (the inspector wire shape carries a `NodeId` as a JSON number). This is - /// the *only* place the numeric value - /// is observed — internal logic compares ids by equality, never by value — - /// so it is exposed solely so a client can round-trip a handle, not to give - /// the value any in-compiler meaning. - pub fn as_u64(self) -> u64 { - self.0 - } } impl std::fmt::Debug for NodeId { @@ -83,9 +77,11 @@ impl std::fmt::Debug for NodeId { // Wire shape (inspector, feature `serde`): a bare JSON number. A `NodeId` is an // opaque handle the client round-trips, so it serializes as its underlying -// `u64` (the one place the numeric value is observed — see [`NodeId::as_u64`]), -// not as a struct. Hand-written rather than `#[serde(transparent)]` because the -// inner field is private. +// `u64`, read off the field directly, not as a struct. Hand-written rather than +// `#[serde(transparent)]` because the inner field is private. There is no +// accessor for the number: nothing in the compiler reads it — ids are compared +// by equality — so the wire impl is its only reader, and an accessor can come +// back when a caller needs one. // // TODO(wire-stability): the mint-order value is not stable across compiler // changes — anything that shifts upstream mint *counts* renumbers every later id, @@ -108,6 +104,11 @@ impl serde::Serialize for NodeId { /// Minimal on purpose: only the stages that *mint or restructure* expression /// nodes (and therefore need to record why a node exists) appear here. /// +/// **Declaration order is not pipeline order**: `Channelize` is declared ahead of +/// `Transact` and `Letrec`, which run before it. The derived [`Ord`] is for map +/// keys, and nothing sorts passes by it — the pipeline order is the one in the +/// module docs. +/// /// `Pass` and [`crate::ccl::names::SyntheticKind`] (which tracks *binder* /// provenance: `Pair`, `Mono`, `SolverArg`, …) are deliberately separate enums, /// neither wrapping the other — one tags `NodeId`s (expression nodes), the @@ -118,37 +119,41 @@ impl serde::Serialize for NodeId { pub enum Pass { /// Lowering CHL source into CCL. Lower, - /// The 1:1 binder rename in [`crate::ccl::uniquify`]. + /// The 1:1 binder rename in [`crate::ccl::uniquify`]. **Never + /// constructed**: `uniquify` preserves every node id, so it has nothing to + /// record. The variant exists so the axis covers every pass. Uniquify, /// UDF inlining + beta-reduction ([`crate::ccl::inline`]): the pass that - /// runs between the post-inference and post-desugar snapshots. Mostly + /// runs between the post-inference and post-channelize snapshots. Mostly /// id-preserving (a rebuilt node carries its input id); its genuine /// deviations are the fan-out clones at multi-use call sites (`Copy`s) and /// the wrappers/redexes it drops (`Transform` discards). Inline, - /// Defer desugaring ([`crate::ccl::channelize`]): channelizing + /// Channelization ([`crate::ccl::channelize`]): channelizing /// `Defer`/`Feed`/`Define` into collection unions and contribution records. /// Mostly a 1:1 transform (ids preserved), but its channelization machinery /// synthesizes new nodes (channel unions, contribution records, floated - /// lambdas, DI wrappers) that are tagged `{via: Desugar, nature: Machinery}`. - Desugar, + /// lambdas, DI wrappers) that are tagged `{via: Channelize, nature: Machinery}`. + Channelize, /// The transaction slice of the unified mutability phase /// ([`crate::ccl::transact_phase::run`]): stripping `with begin():` writer /// sites and assembling the `get_prev_txn`-guarded `LetRec` (histories, - /// commit records, taps). Runs between the post-inference and post-desugar - /// snapshots (after `Inline`, before `Desugar`). + /// commit records, taps). Runs between the post-inference and post-channelize + /// snapshots (after `Inline`, before `Channelize`). Transact, /// The induction slice of the unified mutability phase /// ([`crate::ccl::mut_elim::run`]): folding direct-mirror `For`/`MutWrite` /// loops into guarded `LetRec` induction histories. Runs between the - /// post-inference and post-desugar snapshots (after `Transact`, before - /// `Desugar`). + /// post-inference and post-channelize snapshots (after `Transact`, before + /// `Channelize`). Letrec, /// Monomorphization: cloning a generalized definition's subtree once per /// distinct resolved type (during inference). Mono, /// Lambda elimination: synthesizing point-free combinators (`Compose`, - /// `Zip`, `Id`) from explicit lambdas. + /// `Zip`, `Id`) from explicit lambdas. **Never constructed**: `lambda_elim` + /// opens no recording, so no row carries this tag. The variant exists so the + /// axis covers every pass. LambdaElim, /// Join/dataflow planning: hash-join and restrict scaffolding, clause /// fusion, refinement-predicate compilation. diff --git a/src/ccl/simplify.rs b/src/ccl/simplify.rs index 0a40a9a9..bdac181b 100644 --- a/src/ccl/simplify.rs +++ b/src/ccl/simplify.rs @@ -278,9 +278,22 @@ fn apply_simplification_rules(expr: &mut Expr, contains_iteration: bool) -> bool // iteration sources, so they fire regardless of any `iterate` present. // (They also preserve the subtree's iteration set, so `contains_iteration` // — computed before they run — is still accurate at the guard below.) - changed |= check(try_compose_identity(expr), expr); - changed |= check(try_flatten_compose(expr), expr); - changed |= check(try_string_add_to_concat(expr), expr); + changed |= check( + ruled("simplify.compose_identity", expr, try_compose_identity), + expr, + ); + changed |= check( + ruled("simplify.flatten_compose", expr, try_flatten_compose), + expr, + ); + changed |= check( + ruled( + "simplify.string_add_to_concat", + expr, + try_string_add_to_concat, + ), + expr, + ); // Rules that may discard or restructure sub-expressions. Equationally // valid only on pure CCC morphisms, so they must not touch a sub-tree @@ -291,21 +304,87 @@ fn apply_simplification_rules(expr: &mut Expr, contains_iteration: bool) -> bool // what lets the rule set run correctly at any point in the pipeline — the // invariant is a property of the *nodes*, not of pass timing. if !contains_iteration { - changed |= check(try_const_reduce(expr), expr); - changed |= check(try_product_beta_fst(expr), expr); - changed |= check(try_product_beta_snd(expr), expr); - changed |= check(try_literal_tuple_projection(expr), expr); - changed |= check(try_ccc_universal(expr), expr); - changed |= check(try_exponential_beta(expr), expr); - changed |= check(try_exponential_eta(expr), expr); - changed |= check(try_const_apply(expr), expr); - changed |= check(try_product_eta(expr), expr); - changed |= check(try_zip_distribute_compose(expr), expr); + changed |= check(ruled("simplify.const_reduce", expr, try_const_reduce), expr); + changed |= check( + ruled("simplify.product_beta_fst", expr, try_product_beta_fst), + expr, + ); + changed |= check( + ruled("simplify.product_beta_snd", expr, try_product_beta_snd), + expr, + ); + changed |= check( + ruled( + "simplify.literal_tuple_projection", + expr, + try_literal_tuple_projection, + ), + expr, + ); + changed |= check( + ruled("simplify.ccc_universal", expr, try_ccc_universal), + expr, + ); + changed |= check( + ruled("simplify.exponential_beta", expr, try_exponential_beta), + expr, + ); + changed |= check( + ruled("simplify.exponential_eta", expr, try_exponential_eta), + expr, + ); + changed |= check(ruled("simplify.const_apply", expr, try_const_apply), expr); + changed |= check(ruled("simplify.product_eta", expr, try_product_eta), expr); + changed |= check( + ruled( + "simplify.zip_distribute_compose", + expr, + try_zip_distribute_compose, + ), + expr, + ); } changed } +/// Run one rewrite rule under a recording ([`lineage::enter`](crate::ccl::lineage::enter)) +/// keyed on the node the rule is about to rewrite. +/// +/// This is the whole of simplify's provenance instrumentation: **one combinator, +/// applied uniformly to all thirteen rules**, and no rule body changes at all. +/// That is possible because a recording declares nothing — it names the node in +/// the slot and lets the construction hooks report the rest. In particular: +/// +/// * A rule that does not fire mints nothing and the recording is a **preserve**: +/// it records nothing, so wrapping every *attempt* rather than every *firing* +/// costs one push/pop and no log entry. Nothing here needs to know whether the +/// rule fired, which is why the `bool` return is not consulted. +/// * A rule that mutates in place without minting (`try_string_add_to_concat`'s +/// `*op = BinOpKind::Concat`) is likewise a preserve — the node keeps its +/// identity, so its lineage is the self-edge it already had. +/// * A rule that replaces the slot wholesale (`*expr = Expr::compose(flat)`) +/// mints, and those mints record `expr`'s pre-rule id as their parent. +/// * A rule that promotes an existing child into the slot +/// (`*expr = elts.swap_remove(i)`) mints nothing and copies nothing: a +/// preserve again, and correctly so — the promoted node keeps its own id and +/// is its own self-edge, while the discarded siblings are dead by the live-set +/// difference at the boundary. No site says so. +/// +/// `Nature::Machinery`: an algebraic simplification has no source counterpart. +fn ruled( + label: crate::ccl::lineage::RewriteLabel, + expr: &mut Expr, + rule: impl FnOnce(&mut Expr) -> bool, +) -> bool { + let _g = crate::ccl::lineage::enter( + expr.node_id(), + label, + crate::ccl::lineage::Nature::Machinery, + ); + rule(expr) +} + fn check(changed: bool, expr: &Expr) -> bool { // Only re-typecheck when a rewrite actually fired: the assertion validates // that a *transformation* preserved typing, so an untouched expression diff --git a/src/ccl/subst.rs b/src/ccl/subst.rs index 83f24b3a..cb7972d3 100644 --- a/src/ccl/subst.rs +++ b/src/ccl/subst.rs @@ -35,7 +35,7 @@ //! one keeps sharing the source's `Rc` (pointer-equal, so the source context //! stays intact and the common case allocates nothing). //! * **In-place rewrite** ([`Subst::rewrite_expr`]) mutates the *term tree* the -//! caller owns (lambda elimination, inlining, defer desugaring, lowering's +//! caller owns (lambda elimination, inlining, channelization, lowering's //! uncurrying, and the mutability-elimination phases' read-your-writes //! environments — see [`Subst::discharge_env_in_place`]). A predicate the //! substitution actually touches is rebuilt as a @@ -82,12 +82,16 @@ pub enum Mapping { /// `binder ↦ term` — plug a term in for the binder. No inverse. /// (Boxed: a term is much larger than a binder name.) /// - /// **Considered and deferred: `Rc`.** The payload is a template, - /// never a tree node and cloned afresh at every read ([`Mapping::as_expr`]), - /// so sharing it is sound. The solver copies substitutions constantly - /// (`Bound::render_subst`, [`Subst::then`], `compact`, `constrain`): with a - /// `Box` each of those ~28 sites deep-copies the payload tree, with an `Rc` - /// they are refcount bumps. + /// **Considered and deferred: `Rc`.** The payload is a + /// *template* — never a tree node, cloned afresh at every read + /// ([`Mapping::as_expr`]) — so sharing it is sound, and the solver copies + /// substitutions constantly (`Bound::render_subst`, [`Subst::then`], + /// `compact`, `constrain`). With a `Box`, every one of those ~28 sites deep- + /// copies the payload tree; with an `Rc` they are refcount bumps. That cost + /// is **pre-existing** — the derived `Clone` paid it too — so it is an + /// improvement over both arms rather than anything the freshening `Clone` + /// introduced, and it wants its own before/after rather than riding along + /// here. /// /// One trap if it is ever done: [`Subst::for_each_discharge_term_mut`] would /// become `Rc::make_mut`, which copies out through `TypedExpr`'s freshening @@ -109,13 +113,14 @@ pub enum Mapping { /// So copying the map itself must mint nothing: the template is never in a tree, /// and no two nodes can end up sharing an id because of it. /// -/// Without this, every `Subst` copy inherits the freshening and re-mints its -/// payloads. The solver copies substitutions constantly — `Bound::render_subst`, -/// [`Subst::then`], `compact`, `constrain` — and a bound edge's payloads are **type-domain** -/// terms whose ids no step ever produced, so each such copy records a `Copy` -/// against an origin the log never saw and the pane fold reports it as -/// [`Leak::CopyOfUnknown`](crate::ccl::lineage::Leak::CopyOfUnknown). Measured on -/// `generator_pipeline`: 200 of them at the first pane boundary. +/// Without this, the derived `Clone` inherits the freshening and every `Subst` +/// copy re-mints its payloads. That is not merely wasteful. The solver copies +/// substitutions constantly — `Bound::render_subst`, [`Subst::then`], +/// `compact`, `constrain` — and a bound edge's payloads are **type-domain** +/// terms whose ids are outside the recorded id domain, so each such copy records +/// a `Copy` against an origin the table never saw and the pane fold reports it as +/// [`Leak::ParentUnknown`](crate::ccl::lineage::Leak::ParentUnknown). Measured on +/// `generator_pipeline`: 200 of them across the first pane relation. /// /// [`as_expr_preserving`]: Mapping::as_expr_preserving impl Clone for Mapping { @@ -182,41 +187,15 @@ impl Mapping { /// overwritten: a mint fires `on_mint`, and an id no node ends up carrying is /// a phantom birth in the lineage log. /// - /// A `Discharge` is the crate's one copy that shares an id; the literal below - /// carries why. + /// A `Discharge` copies through [`clone_at`](TypedExpr::clone_at), which + /// builds the replacement's root directly at `node_id` and freshens the + /// interior. Neither shape mints an id that no node ends up carrying. fn as_expr_preserving(&self, node_id: NodeId, occurrence_ty: &Type) -> TypedExpr { let out = match self { // See [`as_expr`]: the rename keeps the occurrence's type. Mapping::Rename(to) => TypedExpr::preserve(node_id, TypedExprNode::Var(to.clone())) .with_ty(occurrence_ty.clone()), - // The root takes the occurrence's id, the interior freshens - // (`node.clone()` reaches each child's own `Clone`): N occurrences give - // N subtrees under N ids the tree already holds. The id is what - // attribution resolves through, and a lowered parameter use is the case - // that shows it — uncurry substitutes a machine-made tuple projection - // into every use of `a` in `def add(a, b): a + a + b`, so a freshened - // root resolves through that template, whose span is the whole `def` - // and whose nature is machinery, and all three uses report the header - // instead of their own columns. - // - // A literal rather than `preserve(node_id, …).with_ty(…)`: the - // exhaustive field check is what keeps `user_annotation` from being - // silently dropped. - // - // TODO(subst-lineage): the edge encoding reproduces this entry, span - // and nature alike — freshen the root and record the copy against the - // occurrence instead of the template. It costs a death per occurrence, - // and the record has to land in the enclosing frame, since a nested one - // flushes first (guards drop LIFO) and would order these edges ahead of - // the copy that introduced the template they read. Revisit when the - // pane-level table lands and the two become distinguishable from - // outside this function. - Mapping::Discharge(t) => TypedExpr { - ty: t.ty.clone(), - node: t.node.clone(), - user_annotation: t.user_annotation.clone(), - node_id, - }, + Mapping::Discharge(t) => t.clone_at(node_id), }; assert_preserves_typedness(&out, occurrence_ty); out @@ -484,7 +463,21 @@ impl Subst { /// catch. (That check accordingly demotes to a debug assert — see /// `check_scope_valid`.) pub fn apply_expr(&self, e: &TypedExpr) -> TypedExpr { + // Both early returns build a term the caller owns while `e` stays live in + // whatever tree holds it, so the result is a genuinely new node even + // though the substitution changed nothing. It freshens and is **recorded** + // as a copy of `e`, rather than keeping `e`'s ids: preserving would put + // one id-set on two live terms, which is the defect predicate rebuilding + // just had to be fixed for. + // + // Recording rather than simply freshening is the load-bearing half — + // unbracketed, these produced 50 `ParentUnknown` on `inner_join`. if self.is_id() { + let _g = crate::ccl::lineage::enter( + e.node_id(), + "subst.vacuous", + crate::ccl::lineage::Nature::Machinery, + ); return e.clone(); } // No-op short-circuit: if none of the substituted binders occur free @@ -494,6 +487,11 @@ impl Subst { // path, which is what keeps vacuous transport from copying terms or // rebuilding predicate terms into fresh `Rc`s. if !self.0.keys().any(|k| is_free(k, e)) { + let _g = crate::ccl::lineage::enter( + e.node_id(), + "subst.vacuous", + crate::ccl::lineage::Nature::Machinery, + ); return e.clone(); } self.apply_expr_inner(e) @@ -501,6 +499,17 @@ impl Subst { fn apply_expr_inner(&self, e: &TypedExpr) -> TypedExpr { use TypedExprNode::*; + // Transport mode *builds*: it returns a new `TypedExpr` rather than + // editing one already in the tree (that is `rewrite_expr_go`, which takes + // `&mut` and installs the replacement at the occurrence's own id). So the + // nodes below are genuinely new and want **recording**, not id-preserving + // — and the node they are derived from is `e`. The recording opens at + // function entry because the `Var` arm returns early. + let _g = crate::ccl::lineage::enter( + e.node_id(), + "subst.transport", + crate::ccl::lineage::Nature::Machinery, + ); let node = match &e.node { Var(n) => match self.0.get(n) { // The replacement carries its own type/annotation, so return it @@ -517,11 +526,11 @@ impl Subst { }, // The target name is a *use* of the defer-handle binder (these - // nodes exist only pre-desugar; transport runs during inference, + // nodes exist only pre-channelize; transport runs during inference, // but the uniform engine handles them for the pre-inference // ports). A var-shaped mapping renames the handle; a discharge // to a non-variable term has no Feed/Define shape to land in, so - // the stale handle is kept for desugar's own + // the stale handle is kept for channelize's own // `UnboundDeferHandle` boundary to report (feeding a lambda // parameter is user-reachable: `\d, v -> d << v`). Feed { name, value } => { @@ -696,14 +705,18 @@ impl Subst { /// Discharge `binder ↦ term` over `e` **in place**, cloning `term` only /// when `binder` actually occurs free in `e`. A vacuous substitution /// costs one [`is_free`] walk and **no clone** — the pass-level callers - /// (lambda elimination, defer desugaring, lowering's uncurrying) + /// (lambda elimination, channelization, lowering's uncurrying) /// substitute into many subtrees that never mention the binder, so /// cloning `term` for those would be pure waste. pub fn discharge_in_place(e: &mut TypedExpr, binder: &Name, term: &TypedExpr) { if !is_free(binder, e) { return; } - Subst::discharge(binder.clone(), term.clone()).rewrite_expr(e); + // `term` is a *template*: `as_expr` clones it afresh at every occurrence + // it fills, and that read is where each sibling is minted. Copying it + // into the map must therefore mint nothing — a freshening clone here + // builds one whole extra tree per call that no occurrence ever uses. + Subst::discharge(binder.clone(), term.clone_preserving_ids()).rewrite_expr(e); } /// Discharge a whole **environment** `{name ↦ term, …}` over `e` — every name @@ -772,6 +785,21 @@ impl Subst { // construction. let occurrence_ty = e.ty.clone(); *e = repl.as_expr_preserving(e.node_id, &occurrence_ty); + if !matches!(e.node, TypedExprNode::Var(_)) { + // Compound replacement. `Clone` freshens, so the replacement's + // interior arrives already distinct from the template — what the + // old `freshen_interior_node_ids()` call did explicitly. Type + // slots are out of the id domain, so the predicate `Rc`s the + // clone shares with its source stay shared. + // + // The root is the one place this costs anything: the clone mints + // an id for it and the carry above immediately discards that id + // in favour of the occurrence's. So the occurrence stays in the + // live set (the carry precedes nothing that could displace it), + // and what is stranded is a single minted id that folds as a + // death rather than a defect — one row and one id per + // substituted occurrence, no extra hop. + } return; } // *Every* type slot the node carries, not just `ty` and the annotation: a @@ -1016,7 +1044,23 @@ impl Subst { // collection into a refined domain). Strip the neutral marker so the // predicate stays marker-free — otherwise it churns under `simplify` // and diverges from inference's pre-marker copy. - let new_pred = strip_iterate_markers(&restricted.apply_expr(&r.predicate)); + // **Recorded, not preserved.** `born` below installs a *new* `Rc` while + // the source refinement stays alive behind `r`, so the two terms coexist + // — this is a derivation, not the in-place replacement `PredMemo::rebuild` + // performs. Preserving here put the same ids on two simultaneously-live + // predicate terms, which nothing catches because predicate uniqueness is + // not asserted; measured at 53 such collisions on `inner_join` alone. + // + // The slot is the source predicate's own root, so the rewritten term rows + // as derived from the term it was substituted out of. + let new_pred = { + let _g = crate::ccl::lineage::enter( + r.predicate.node_id(), + "subst.force_refinement", + crate::ccl::lineage::Nature::Machinery, + ); + strip_iterate_markers(&restricted.apply_expr(&r.predicate)) + }; // Scope-validity (design §6.2): a discharged binder must not survive // in the rewritten predicate — once `[x ↦ arg]` fires, no free `x` // may remain, or a downstream pass would observe a dangling @@ -1847,7 +1891,7 @@ mod rewrite_tests { } // Feed/Define handles rename through var-shaped mappings and survive a - // non-variable discharge for desugar's own boundary to diagnose. + // non-variable discharge for channelize's own boundary to diagnose. #[test] fn rewrite_renames_feed_handles() { let mut e = TypedExpr::feed("d", var("d")); diff --git a/src/ccl/transact_phase.rs b/src/ccl/transact_phase.rs index 9ed36788..9b4dcbf3 100644 --- a/src/ccl/transact_phase.rs +++ b/src/ccl/transact_phase.rs @@ -79,14 +79,16 @@ use crate::ccl::{ BaseType, Builtin, Expr, F_DECISION, F_TIME, F_WRITE, F_WRITE_TARGETS, F_WRITES, FieldKey, HistoryKind, Lit, Name, ProjKey, Type, TypedBinding, TypedExprNode, WriterSite, ccl_utils::{free_names_in_value, is_free_in_value, synthesize_arm_predicate}, + lineage, mut_elim::{close_recurrence_group, fold_induction_loop, hoist_feeds, mut_var_value_tys}, + provenance::NodeId, subst::Subst, }; /// Recognize a **fed-out mutable variable read** and rewrite it to an as-of join, *before* /// lambda elimination. Run after `channelize`. /// -/// After defer-desugaring, a read-only reply is a chain of mutable variable reads feeding a +/// After channelization, a read-only reply is a chain of mutable variable reads feeding a /// broadcast over a reading loop: /// `let k₁ = final_or_default((balance.f₁, _)) in … let kₙ = … in trigger ≫ (λ r → e)`, /// where `e` reads the `kᵢ` and `mutable variable` is a commit log (`Txn`, a non-enumerable @@ -146,8 +148,32 @@ fn rewrite_as_of_reads_go(expr: &mut Expr) { // outer read binding captures the chain rather than the innermost `let` firing a // single-variable rewrite in isolation (which would strand the outer reads // unresolved). - if let Some(rewritten) = as_of_join(expr) { - *expr = rewritten; + // + // The slot is that outermost `let`, because the whole chain it heads is what + // the as-of join replaces: the `zip`/`as_of` scaffolding, the rebuilt reply + // lambda, and the snapshot projections all stand in for it, and the inner + // `let`s of the chain die with it (deaths are the boundary difference, so + // nothing names them). `Nature::Expansion` — `resp << balance` really is an + // as-of read; the join is the faithful rendering of what the user wrote, not + // plumbing. + // + // Recording the *attempt* rather than the firing is free: a non-matching + // node mints nothing, so the recording writes nothing. The walk below sits + // outside the recording, so a nested chain attributes to its own `let`. + // + // These rows reach no table in a normal compile: `compile_program` calls + // `rewrite_as_of_reads` outside every pass scope it opens, so they land only + // under an audit window (`CAMBRA_LINEAGE_AUDIT=full`, which ends at + // `post-as-of-read`). + { + let _g = lineage::enter( + expr.node_id(), + "transact.as_of_read", + lineage::Nature::Expansion, + ); + if let Some(rewritten) = as_of_join(expr) { + *expr = rewritten; + } } expr.walk_children_mut(rewrite_as_of_reads_go); } @@ -545,6 +571,12 @@ fn mut_var_value_ty(ty: &Type) -> Type { /// A stripped `with begin():` writer site, before its decision body is built. struct RawSite { + /// The statement node the `with begin():` block was stripped from — the + /// lineage slot every node built for this site parents on. Carried on the + /// site rather than re-derived because the block is *disassembled* on the way + /// to a writer: by [`build_writer`] the `Begin` and its `ExprStmt` are gone, + /// and the decision body is the only surviving piece. + slot: NodeId, /// The loop item binder (`for r in xs`); the synthetic singleton binder for a /// standalone transaction. target: TypedBinding, @@ -594,6 +626,16 @@ struct FeedSite { /// [`collect_txn_mut_vars`]). Keying on the exact binder identity (not the surface /// base name) makes the fold immune to an unrelated local variable merely /// *spelled* like a mutable variable. +/// One commit store's writer sites and their in-block feeds, as [`run`] collects +/// them before the store is planned. +/// +/// Each writer travels with the [`NodeId`] of the `with begin():` statement its +/// block was stripped from — the site's lineage slot, which rides beside +/// [`WriterSite`] rather than on it because that type is shared IR that planning +/// rebuilds from a `Transact` carrier. `site_feeds[j]` holds site `j`'s feeds, so +/// the two vectors stay index-parallel. +type StoreWriters = (Vec<(NodeId, WriterSite)>, Vec>); + pub fn run(expr: Expr, txn_mut_vars: &HashSet) -> Result { // Strip whenever a `with begin():` block is present. A block need not write a // transactional mutable variable — a read-only block (`out << balance`) has no write @@ -663,9 +705,9 @@ pub fn run(expr: Expr, txn_mut_vars: &HashSet) -> Result { return Ok(stripped); } - // Each key's tick-0 `init`, located at its `MutDecl` (the value type is - // the init's type — the snapshot/write element type of that variable). - let mut key_init: HashMap = HashMap::new(); + // Each key's tick-0 `init`, located at its `let` binding (the value type is + // the init's type — the snapshot/write element type of that register). + let mut key_init: HashMap = HashMap::new(); collect_key_inits(&stripped, &key_names, &mut key_init); for k in &key_names { assert!( @@ -715,17 +757,22 @@ pub fn run(expr: Expr, txn_mut_vars: &HashSet) -> Result { // the right store with no await-specific logic, and the shared // `let k = as_of_read(…)` rebind is left to serve the as-of reads only. for _ in 0..=key_names.len() { - if !key_init.values().any(contains_await_final) { + if !key_init.values().any(|d| contains_await_final(&d.init)) { break; } for k in key_names.clone() { - let mut init = key_init[&k].clone(); + // The rewritten seed *replaces* the stash, so the copy Rust forces here + // is a move rather than a duplication: preserve the ids, which the + // stash already recorded against the key's `MutDecl`. The terminal + // reads the rewrite mints are recorded inside `resolve_await_finals`. + let mut init = key_init[&k].init.clone_preserving_ids(); resolve_await_finals(&mut init, &hist, &key_init); - key_init.insert(k, init); + let decl = key_init[&k].decl; + key_init.insert(k, MutVarDecl { decl, init }); } } assert!( - !key_init.values().any(contains_await_final), + !key_init.values().any(|d| contains_await_final(&d.init)), "transact_phase: a mutable variable seed still awaits after one resolution round per key — \ the await relation on seeds must be acyclic" ); @@ -751,13 +798,21 @@ pub fn run(expr: Expr, txn_mut_vars: &HashSet) -> Result { // per-store so two stores' taps cannot collide either. Feeds are kept *per site* // (parallel to `writers`) so each tap binding reads its own commit-record stream. let mut feed_counter = 0usize; - let mut per_store: Vec<(Vec, Vec>)> = (0..groups.len()) + // Each writer travels with the statement node its block was stripped from, so + // `plan_store` can parent that site's commit record on it. The slot rides + // beside `WriterSite` rather than on it: `WriterSite` is shared IR that + // planning rebuilds from a `Transact` carrier, and a lineage slot is not a + // fact about the carrier. + let mut per_store: Vec = (0..groups.len()) .map(|_| (Vec::new(), Vec::new())) .collect(); for s in sites { let store = store_of(&s); + let slot = s.slot; + let g = lineage::enter(slot, "transact.writer", lineage::Nature::Expansion); let (writer, feeds) = build_writer(s, &key_init, &mut feed_counter, &cross.acc_views); - per_store[store].0.push(writer); + drop(g); + per_store[store].0.push((slot, writer)); per_store[store].1.push(feeds); } @@ -867,6 +922,13 @@ fn partition_keys( #[derive(Default)] struct CrossDomain { bindings: Vec<(TypedBinding, Expr)>, + /// The `ExprStmt` each `bindings` entry was folded out of, index-parallel + /// with it — the lineage slot for the letrec [`wrap_cross_domain`] builds + /// around that binding. It rides beside `bindings` rather than inside for + /// the same reason a writer's slot rides beside its `WriterSite`: the + /// binding is IR that recognition rebuilds, and a slot is not a fact about + /// it. + slots: Vec, reads: Vec<(TypedBinding, Expr)>, feeds: Vec<(Name, Expr)>, acc_views: HashMap, @@ -915,6 +977,7 @@ fn fold_cross_domain_loops(expr: Expr, cross_reads: &HashSet, out: &mut Cr && let TypedExprNode::For { body, .. } = &effect.node && loop_writes_any(body, cross_reads) { + let stmt_id = expr.node_id(); let TypedExprNode::ExprStmt { expr: effect, body: cont, @@ -922,9 +985,21 @@ fn fold_cross_domain_loops(expr: Expr, cross_reads: &HashSet, out: &mut Cr else { unreachable!("guarded above") }; + let effect_id = effect.node_id(); let TypedExprNode::For { target, iter, body } = effect.node else { unreachable!("guarded above") }; + // The statement is the slot: everything the fold builds stands in for it, + // and it leaves the tree entirely. `blame` names the `For` so the products + // resolve to the loop keyword's span rather than the statement's. Both + // choices mirror `mut_elim`'s `letrec.loop`, which records the *same* + // `fold_induction_loop` call for a loop that stays in that pass. + let g = lineage::enter( + stmt_id, + "transact.cross_domain_fold", + lineage::Nature::Expansion, + ); + g.blame(&[effect_id]); // The loop body and the continuation between them carry every reference to // this loop's accumulators, so their `Mut(V, D)`s give each one the value // type inference joined for it. @@ -948,11 +1023,13 @@ fn fold_cross_domain_loops(expr: Expr, cross_reads: &HashSet, out: &mut Cr ); } } + drop(g); let mut cont = *cont; for (acc, x_final) in &fold.renames { rename_var_uses(&mut cont, acc, x_final); } out.bindings.push(fold.binding); + out.slots.push(stmt_id); out.reads.extend(fold.reads); out.feeds.extend(fold.feed_views); return fold_cross_domain_loops(cont, cross_reads, out); @@ -1045,6 +1122,11 @@ fn strip( if let TypedExprNode::ExprStmt { expr: effect, .. } = &expr.node && matches!(&effect.node, TypedExprNode::Begin { .. }) { + // The statement node the block hangs off — the slot for everything this + // strip and the writer build downstream of it mint. The `Begin` marker + // is the blamed node, so products resolve to the `with` keyword's span + // rather than the enclosing statement's. + let stmt_id = expr.node_id; let TypedExprNode::ExprStmt { expr: effect, body: rest, @@ -1052,6 +1134,7 @@ fn strip( else { unreachable!("guarded above") }; + let begin_id = effect.node_id; let TypedExprNode::Begin { body: block } = effect.node else { unreachable!("guarded above") }; @@ -1067,17 +1150,30 @@ fn strip( "a writing `with begin():` block must be inside a loop (lowering wraps a \ standalone block in a singleton `For`)", ); - let (txn_block, lifted) = partition_block(*block, txn_mut_vars); - let (read_keys, write_keys) = collect_footprint(&txn_block, txn_mut_vars); - out.sites.push(RawSite { - target: target.clone(), - source: source.clone(), - block: txn_block, - read_keys, - write_keys, - enclosing_writes: enclosing_writes.clone(), - }); - let new_rest = prepend_effects(lifted, *rest); + let new_rest = { + // Recorded here: the source copy the site takes, the statement + // wrappers `prepend_effects` mints for the lifted induction + // writes, and whatever `partition_block` rebuilds. The block + // itself is *not* consumed here — it travels on the site and is + // disassembled by `build_writer` under this same slot. + let g = lineage::enter(stmt_id, "transact.strip", lineage::Nature::Expansion); + g.blame(&[begin_id]); + let (txn_block, lifted) = partition_block(*block, txn_mut_vars); + let (read_keys, write_keys) = collect_footprint(&txn_block, txn_mut_vars); + out.sites.push(RawSite { + slot: stmt_id, + target: target.clone(), + // The enclosing `For` keeps its `iter` in the stripped tree while + // the site carries the same expression into the writer's source, + // so the site's copy is its own. + source: source.clone(), + block: txn_block, + read_keys, + write_keys, + enclosing_writes: enclosing_writes.clone(), + }); + prepend_effects(lifted, *rest) + }; return strip(new_rest, txn_mut_vars, enclosing, out); } // A read-only block (feeds a mutable variable read, no txn write) → unwrap it onto @@ -1089,7 +1185,16 @@ fn strip( if reads.len() > 1 { out.read_only_footprints.push(reads); } - let spliced = splice_block(*block, *rest); + let spliced = { + // `splice_block` re-types the spine as it re-points each statement's + // continuation, but preserves every id, so this recording usually + // captures nothing and writes nothing. It is here because the arm is a + // rewrite: if a re-typed rebuild ever starts minting, the node lands + // on the statement it belongs to instead of becoming a leak. + let g = lineage::enter(stmt_id, "transact.unwrap_block", lineage::Nature::Machinery); + g.blame(&[begin_id]); + splice_block(*block, *rest) + }; return strip(spliced, txn_mut_vars, enclosing, out); } // A `For`: thread it as the enclosing loop for its body (its source is @@ -1529,7 +1634,7 @@ pub fn check_await_final_linearity(expr: &Expr) -> Result<(), String> { fn check_store_acyclicity( stripped: &Expr, sites: &[RawSite], - key_init: &HashMap, + key_init: &HashMap, groups: &[Vec], ) -> Result<(), String> { /// Every awaited key `e` depends on, directly or through a marked binder. @@ -1608,7 +1713,7 @@ fn check_store_acyclicity( // In key order — `groups` is built in `key_names` order — rather than `key_init`'s // hash order, so a program with two clashing seeds names the same one every run. for k in groups.iter().flatten() { - if let Some(j) = clash(&awaited_in(&key_init[k], &marked), store_of(k)) { + if let Some(j) = clash(&awaited_in(&key_init[k].init, &marked), store_of(k)) { return Err(format!( "the seed of transactional mutable variable `{}` depends on `await_final({})`, and the \ two share a commit store — `{}`'s value at commit tick 0 would await that \ @@ -1668,15 +1773,24 @@ fn resolve_writer_free_awaits(e: &mut Expr, written_keys: &[Name]) { if awaited.is_empty() { return; } - let mut seeds: HashMap = HashMap::new(); + let mut seeds: HashMap = HashMap::new(); collect_key_inits(e, &awaited, &mut seeds); - fn rewrite(e: &mut Expr, seeds: &HashMap) { + fn rewrite(e: &mut Expr, seeds: &HashMap) { if let TypedExprNode::Apply { argument, function } = &e.node && matches!(&function.node, TypedExprNode::Builtin(Builtin::AwaitFinal)) && let TypedExprNode::Var(reg) = &argument.node && let Some(seed) = seeds.get(reg) { - *e = seed.clone(); + // The marker node is what the seed stands in for, and it is user-written + // (`await_final(x)` is source text), so it is the slot. The key's + // `MutDecl` stays on the spine here — this is the writer-free case — so + // the seed's original is still live and the copy must freshen. + let _g = lineage::enter( + e.node_id(), + "transact.await_final_seed", + lineage::Nature::Expansion, + ); + *e = seed.init.clone(); return; } e.walk_children_mut(|c| rewrite(c, seeds)); @@ -1852,15 +1966,15 @@ fn proj_tuple(p: &Name, tuple_ty: &Type, i: usize, elt_ty: Type) -> Expr { /// on the disjunction of any `if` guards' write paths. fn build_writer( site: RawSite, - key_init: &HashMap, + key_init: &HashMap, feed_counter: &mut usize, acc_views: &HashMap, ) -> (WriterSite, Vec) { let value_ty = |k: &Name| { key_init .get(k) - .map(|e| mut_var_value_ty(&e.ty)) - .expect("transact_phase: footprint key must be a mutable variable key") + .map(|d| mut_var_value_ty(&d.init.ty)) + .expect("transact_phase: footprint key must be a register key") }; let read_tys: Vec = site.read_keys.iter().map(value_ty).collect(); let orig_item_ty = site @@ -2325,18 +2439,49 @@ fn and_path(path: &Expr, guard: &Expr) -> Expr { e } -/// Locate each mutable variable key's `let` binding and record its tick-0 `init` (keeping -/// the outermost when a key is bound more than once). The `init` carries the -/// key's value type (its `.ty`). -fn collect_key_inits(expr: &Expr, keys: &[Name], out: &mut HashMap) { +/// A register key's `let` declaration, as the phase folds it away. +struct MutVarDecl { + /// The `let` node that declared the register. [`walk_spine`] **drops** it — + /// the key's history binding is what stands in its place — so it is the + /// lineage slot for everything built for this key. + decl: NodeId, + /// The tick-0 initial value, which carries the key's value type (its `.ty`). + /// It reaches the output **once**, as the `get_prev_txn` default: the + /// trailing read is an [`as_of_read`], which carries no seed operand because + /// tick 0 of the store is its keys' seeds. Readers that want only the value + /// type — [`StorePlan::reads`], [`final_key`] — borrow it rather than + /// placing it. + init: Expr, +} + +/// Locate each register key's `let` binding and record it (keeping the outermost +/// when a key is bound more than once). +fn collect_key_inits(expr: &Expr, keys: &[Name], out: &mut HashMap) { if let TypedExprNode::MutDecl { binding, init, .. } = &expr.node && keys.contains(&binding.name) && !out.contains_key(&binding.name) { - // A stash, not a duplication: the `let` that held this init is dropped - // rather than kept alongside, so preserving its ids is what makes the - // later placements copies **of the original**. - out.insert(binding.name.clone(), init.clone_preserving_ids()); + // Stash the register's init so a later stage can place it: `walk_spine` + // drops this whole `MutDecl` for a register key, so the original is gone + // by then. The copy freshens and is **recorded** against the `MutDecl` + // it is taken from — the same node the stash already carries as `decl`, + // and the one the register's later scaffolding is attributed to. + // + // Preserving instead would also be sound (only one of the two is ever + // live), but it saved 20 ids over the whole pipeline suite — subtrees of + // 1 to 3 nodes — which does not pay for an opt-out. + let _g = lineage::enter( + expr.node_id(), + "transact.key_init_stash", + lineage::Nature::Machinery, + ); + out.insert( + binding.name.clone(), + MutVarDecl { + decl: expr.node_id(), + init: (**init).clone(), + }, + ); } expr.walk_children(|c| collect_key_inits(c, keys, out)); } @@ -2378,7 +2523,7 @@ fn apply_ty(arg: Expr, func: Expr, ty: Type) -> Expr { /// A `Builtin(b)` node stamped with its recorded type. The transaction /// oracle/guard builtins ([`Builtin::BeginTxn`] / [`Builtin::GetPrevTxn`]) are /// minted here, **post-inference**, so their type is not inferred: the CHECK-mode -/// `typecheck` between this phase and desugar trusts the recorded type set here, +/// `typecheck` between this phase and channelize trusts the recorded type set here, /// and `recognize` consumes them before op-conversion. `BeginTxn` has no /// inference scheme at all; `GetPrevTxn` does carry one (its guard-accessor /// scheme, symmetric with the induction `GetPrevSeq`), but the current pipeline @@ -2546,8 +2691,8 @@ struct HoistedFeed { fn plan_store( key_names: Vec, all_hist: &HashMap, - key_init: &HashMap, - writers: Vec, + key_init: &HashMap, + writers: Vec<(NodeId, WriterSite)>, site_feeds: Vec>, ) -> StorePlan { let hist: HashMap = key_names @@ -2561,8 +2706,8 @@ fn plan_store( let value_ty = |k: &Name| { key_init .get(k) - .map(|e| mut_var_value_ty(&e.ty)) - .expect("transact_phase: footprint key must be a mutable variable key") + .map(|d| mut_var_value_ty(&d.init.ty)) + .expect("transact_phase: footprint key must be a register key") }; // --- commit-record + tap bindings, one commit binding per writer site --- @@ -2576,7 +2721,12 @@ fn plan_store( let mut site_decision_ty: Vec = Vec::with_capacity(writers.len()); let mut site_write_keys: Vec> = Vec::with_capacity(writers.len()); - for (j, (w, feeds)) in writers.into_iter().zip(site_feeds).enumerate() { + for (j, ((slot, w), feeds)) in writers.into_iter().zip(site_feeds).enumerate() { + // This site's commit record, its tap bindings, and the snapshot + // scaffolding around the writer body all belong to the `with begin():` + // statement they were built for. The writer `body` passes through + // verbatim and keeps its own ids. + let _g = lineage::enter(slot, "transact.commit_record", lineage::Nature::Expansion); let WriterSite { read_keys, write_keys, @@ -2701,10 +2851,22 @@ fn plan_store( // --- history bindings, one per key --- let mut hist_bindings: Vec<(TypedBinding, Expr)> = Vec::with_capacity(key_names.len()); for k in &key_names { + // The key's history binding stands in for its `let` declaration, which + // `walk_spine` drops — so that `let` is the slot, and the merged per-key + // commit view, the `get_prev_txn` application and the wrapping lambda all + // parent on it. + let _g = lineage::enter( + key_init[k].decl, + "transact.history", + lineage::Nature::Expansion, + ); let v = value_ty(k); let reg_k = hist[k].clone(); let t = Name::fresh("__t"); - let init = key_init.get(k).expect("key init present").clone(); + // The init's one placement in the output, as this `get_prev_txn` + // default. The `let` that held the original is dropped by `walk_spine`, + // so this copy stands in for it and rows on the same slot. + let init = key_init.get(k).expect("key init present").init.clone(); // The `get_prev_txn` history slot — the design's denotation: the // `⧺`-merged **per-key commit views** of every site writing this key // ("multiple writer sites for one variable merge their commit @@ -2846,17 +3008,44 @@ impl StorePlan { /// [`resolve_await_finals`] already gave each await its own terminal read, so /// binding an as-of read here too would leave a second, unconsumed read on the /// store. - fn reads(&self, key_init: &HashMap, body: &Expr) -> Vec<(TypedBinding, Expr)> { + fn reads( + &self, + key_init: &HashMap, + body: &Expr, + ) -> Vec<(TypedBinding, Expr)> { self.key_names .iter() .filter(|k| is_free_in_value(k, body)) .map(|k| { - let v = mut_var_value_ty(&key_init[k].ty); + // The as-of read is the key's *second* stand-in for the declaration + // this phase drops (the history binding was the first), so it + // parents on that same `let`. + let _g = lineage::enter( + key_init[k].decl, + "transact.key_rebind", + lineage::Nature::Expansion, + ); + let v = mut_var_value_ty(&key_init[k].init.ty); (binding(k.clone(), v.clone()), as_of_read(&self.hist[k], v)) }) .collect() } + /// The node this store's carrier parents on: the **outermost** declaration among + /// its keys, in first-occurrence order. + /// + /// A store's letrec replaces no single node — it is what those declarations and + /// their writers collectively became — so there is no node it stands in for. The + /// first key's dropped `MutDecl` is the honest answer anyway: it is a real node + /// this phase removed rather than a stand-in, and it is where a reader asking + /// where the transaction structure came from should land. + fn carrier_slot(&self, key_init: &HashMap, tail: &Expr) -> NodeId { + self.key_names + .first() + .map(|k| key_init[k].decl) + .unwrap_or_else(|| tail.node_id()) + } + /// The feed views to hoist over this store's body, in source order. fn feed_views(&self) -> Vec<(Name, Expr)> { self.hoisted @@ -2906,7 +3095,7 @@ impl StorePlan { fn splice_stores( expr: Expr, stores: &[StorePlan], - key_init: &HashMap, + key_init: &HashMap, cross: CrossDomain, ) -> Expr { let free_before = free_names_in_value(&expr); @@ -2937,7 +3126,7 @@ fn splice_stores( fn walk_spine( expr: Expr, stores: &[StorePlan], - key_init: &HashMap, + key_init: &HashMap, cross: CrossDomain, carried: &mut Vec>, ) -> Expr { @@ -2985,12 +3174,24 @@ fn walk_spine( .rev() .fold(inner, |body, node| relink_spine_body(node, body)); let reads = store.reads(key_init, &inner); - inner = close_recurrence_group( - store.bindings.clone(), - reads, - store.feed_views(), - inner, + // The plan's bindings move out of it rather than being duplicated: + // `stores` is borrowed and each plan is placed exactly once, so the + // copy the borrow forces is the only live one. Preserving keeps the + // ids `plan_store` recorded against each site's `with begin():` + // statement and each key's declaration; freshening here would + // re-parent that whole tree on the carrier and throw the finer + // attribution away. + let bindings: Vec<(TypedBinding, Expr)> = store + .bindings + .iter() + .map(|(b, e)| (b.clone(), e.clone_preserving_ids())) + .collect(); + let _g = lineage::enter( + store.carrier_slot(key_init, &inner), + "transact.carrier", + lineage::Nature::Expansion, ); + inner = close_recurrence_group(bindings, reads, store.feed_views(), inner); } match cross { Some(c) => wrap_cross_domain(inner, c), @@ -3128,12 +3329,25 @@ fn relink_spine_body(mut node: Expr, inner: Expr) -> Expr { /// still a footprint key of any site that reads it, and one nothing touches at all is /// resolved earlier by [`resolve_writer_free_awaits`]. The post-condition assert in /// [`run`] confirms none survives. -fn resolve_await_finals(e: &mut Expr, hist: &HashMap, key_init: &HashMap) { +fn resolve_await_finals( + e: &mut Expr, + hist: &HashMap, + key_init: &HashMap, +) { if let TypedExprNode::Apply { argument, function } = &e.node && matches!(&function.node, TypedExprNode::Builtin(Builtin::AwaitFinal)) && let TypedExprNode::Var(reg) = &argument.node && hist.contains_key(reg) { + // The marker is user-written source text, and the terminal read is what it + // becomes, so the marker node is the slot. Its `Var(x)` operand dies with + // it — the read names the history binding, not the key — which the boundary + // difference reports without anything having to declare it. + let _g = lineage::enter( + e.node_id(), + "transact.await_final", + lineage::Nature::Expansion, + ); *e = final_key(reg, hist, key_init); return; } @@ -3148,8 +3362,8 @@ fn resolve_await_finals(e: &mut Expr, hist: &HashMap, key_init: &Has /// here, for its *type* — the value type of the history the read names — and not as a /// term. A key no writer site writes never reaches here at all: /// [`resolve_writer_free_awaits`] has replaced its await with the seed. -fn final_key(k: &Name, hist: &HashMap, key_init: &HashMap) -> Expr { - let init = key_init.get(k).cloned().expect("key init present"); +fn final_key(k: &Name, hist: &HashMap, key_init: &HashMap) -> Expr { + let init = &key_init.get(k).expect("key init present").init; let v = mut_var_value_ty(&init.ty); sampling_read(&hist[k], v, Builtin::FinalRead) } @@ -3185,12 +3399,46 @@ fn wrap_cross_domain(txn_letrec: Expr, cross: CrossDomain) -> Expr { if cross.bindings.is_empty() { return txn_letrec; } + let CrossDomain { + bindings, + slots, + reads, + feeds, + .. + } = cross; + debug_assert_eq!( + bindings.len(), + slots.len(), + "a folded cross-domain binding must carry the statement it came from" + ); let mut inner = txn_letrec; - for (b, def) in cross.reads.into_iter().rev() { - inner = let_typed(b.name, b.ty, def, inner); - } - inner = hoist_feeds(inner, cross.feeds); - for (b, def) in cross.bindings.into_iter().rev() { + { + // The group-level wrappers — each trailing final read and the feed hoists + // — sit in the shared body inside *every* folded loop's letrec, so no + // single loop is their slot. The outermost folded statement is, by the + // argument [`StorePlan::carrier_slot`] makes for the store carrier: it is + // a real node this phase removed, and it is the first of the statements + // this group collectively replaced. + let outermost = slots.first().copied().unwrap_or_else(|| inner.node_id()); + let _g = lineage::enter( + outermost, + "transact.cross_domain_body", + lineage::Nature::Machinery, + ); + for (b, def) in reads.into_iter().rev() { + inner = let_typed(b.name, b.ty, def, inner); + } + inner = hoist_feeds(inner, feeds); + } + // One single-binding letrec per folded loop, each parented on the statement + // that loop was folded out of — the same slot `transact.cross_domain_fold` + // recorded its binding against, so the carrier and its contents agree. + for ((b, def), slot) in bindings.into_iter().zip(slots).rev() { + let _g = lineage::enter( + slot, + "transact.cross_domain_group", + lineage::Nature::Expansion, + ); let ty = inner.ty.clone(); inner = Expr::new(TypedExprNode::LetRec { bindings: vec![(b, def)], @@ -3211,6 +3459,9 @@ mod tests { fn footprint_site(read_keys: &[&Name], write_keys: &[&Name]) -> RawSite { let unit = Expr::new(TypedExprNode::Lit(Lit::Unit)); RawSite { + // A placeholder like the rest: the partition reads footprints only, and + // nothing here records. + slot: unit.node_id(), target: binding(Name::fresh("__r"), Type::Base(BaseType::Unit)), source: unit.clone(), block: unit, diff --git a/src/ccl/ty.rs b/src/ccl/ty.rs index 90086ac4..afbacf2c 100644 --- a/src/ccl/ty.rs +++ b/src/ccl/ty.rs @@ -528,9 +528,9 @@ pub fn reset_kind_var_counter() { /// |---|---|---|---| /// | `Hole` | Lowering | "This slot needs a type; not yet known" | End of inference (compiler bug if survives — flagged as `UnresolvedHole`) | /// | `BoundedHole(𝑇)` | Lowering | "A bounded annotation `𝑥 <: 𝑇`: infer this, subject to `<: 𝑇`" — an obligation, not a shape | Pass 1's `normalize_annotation` (flagged as `UnresolvedBoundedHole` if it survives) | -/// | `Infer(id)` | Type checker only | "Inference variable N from the coalesce pass" | End of inference for any type reachable from the program's root output (flagged as `UnresolvedInfer` by `collect_type_errors`); an induction accumulator's *domain* is necessarily `Infer` until the unified phase resolves it (see `Strictness::PreDesugar`) | +/// | `Infer(id)` | Type checker only | "Inference variable N from the coalesce pass" | End of inference for any type reachable from the program's root output (flagged as `UnresolvedInfer` by `collect_type_errors`); an induction accumulator's *domain* is necessarily `Infer` until the unified phase resolves it (see `Strictness::PreChannelize`) | /// | `History` (`kind: Overwrite`) | Type checker only | "Mutable variable: a `value` cell tracked over a `domain` (loop index or transaction time)" | the unified phase (`transact_phase` / `mut_elim`, which runs *before* `channelize`; a survivor downstream is a compiler bug) | -/// | `History` (`kind: Feed`) | Type checker only | "Feed channel `domain ⇒ value`: the defer binding's post-desugar stream type" | `channelize` (which runs after inference; a survivor downstream is a compiler bug) | +/// | `History` (`kind: Feed`) | Type checker only | "Feed channel `domain ⇒ value`: the defer binding's post-channelize stream type" | `channelize` (which runs after inference; a survivor downstream is a compiler bug) | /// | `ChanDom(d, _)` | Type checker only | "Rigid nominal domain of feed channel `d` — its domain resolves at channel assembly" | `channelize` (substituted to the concrete channel domain; a survivor downstream is a compiler bug) | #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Type { diff --git a/src/ccl/uniquify.rs b/src/ccl/uniquify.rs index 9abfb988..18136a48 100644 --- a/src/ccl/uniquify.rs +++ b/src/ccl/uniquify.rs @@ -77,17 +77,74 @@ use std::collections::HashMap; use crate::ccl::ccl_utils::PredMemo; use crate::ccl::{Expr, Name, Type, TypedBinding, TypedExprNode}; +/// Every **distinct** refinement-predicate term reachable from `expr`, as a +/// multiset of their id-sets, deduped by `Rc` pointer. +/// +/// One predicate term riding many type slots is one term; two entries here mean +/// two genuinely different `Rc`s. Used to assert uniquify's predicate handling is +/// **1:1** — see the tripwire in [`run`]. +#[cfg(debug_assertions)] +fn distinct_predicate_terms(expr: &Expr) -> Vec> { + use crate::ccl::provenance::NodeId; + use std::collections::{HashMap, HashSet}; + + fn ids_of(e: &Expr, out: &mut Vec) { + out.push(e.node_id()); + e.walk_children(|c| ids_of(c, out)); + } + fn from_ty(t: &Type, acc: &mut HashMap>, seen: &mut HashSet) { + if let Type::Refinement(_, r) = t { + let key = std::rc::Rc::as_ptr(&r.predicate) as usize; + if seen.insert(key) { + let mut v = Vec::new(); + ids_of(&r.predicate, &mut v); + v.sort_unstable(); + acc.insert(key, v); + from_expr(&r.predicate, acc, seen); + } + } + t.walk_children(|c| from_ty(c, acc, seen)); + } + fn from_expr(e: &Expr, acc: &mut HashMap>, seen: &mut HashSet) { + from_ty(&e.ty, acc, seen); + if let Some(a) = &e.user_annotation { + from_ty(a, acc, seen); + } + if let TypedExprNode::Cast { target, .. } = &e.node { + from_ty(target, acc, seen); + } + e.walk_children(|c| from_expr(c, acc, seen)); + } + let mut acc = HashMap::new(); + let mut seen = HashSet::new(); + from_expr(expr, &mut acc, &mut seen); + let mut out: Vec> = acc.into_values().collect(); + out.sort(); + out +} + /// α-uniquify every binder in `expr` (see module docs). Runs once per -/// program, immediately after lowering and before defer desugaring. +/// program, immediately after lowering and before channelization. pub fn run(mut expr: Expr) -> Expr { // Snapshot every node's `NodeId` before the rename so we can assert // it survives unchanged (collected only under debug_assertions). #[cfg(debug_assertions)] let before_ids = collect_node_ids(&expr); + // The **1:1 predicate** precondition. Uniquify cannot mutate through a + // predicate's `Rc`, so it rebuilds each one and repoints the refinement it + // was handed. That is only a *replacement* — and preserving the ids only + // honest — if the walk reaches every occurrence, so that no original term + // survives beside its rebuild. Asserted rather than assumed: N distinct + // predicate terms in, N distinct terms out, carrying the same ids. + #[cfg(debug_assertions)] + let before_preds = distinct_predicate_terms(&expr); let mut u = Uniquifier { env: HashMap::new(), - memo: PredMemo::new(), + // Replacing, not deriving: this walk reaches every occurrence of every + // predicate it rebuilds, so no original survives beside its rebuild. The + // tripwire below asserts that 1:1 correspondence on every compile. + memo: PredMemo::replacing(), }; u.expr(&mut expr); debug_assert!( @@ -104,6 +161,22 @@ pub fn run(mut expr: Expr) -> Expr { "uniquify must preserve every NodeId (1:1 in-place rename); \ provenance ids are stable across this pass" ); + let after_preds = distinct_predicate_terms(&expr); + debug_assert_eq!( + before_preds.len(), + after_preds.len(), + "uniquify must be 1:1 on predicate terms: {} distinct terms in, {} out. \ + A mismatch means the walk missed an occurrence, so an original term \ + survives beside its rebuild — and then preserving their ids puts one \ + id-set on two live terms.", + before_preds.len(), + after_preds.len(), + ); + debug_assert_eq!( + before_preds, after_preds, + "uniquify's rebuilt predicate terms must carry the same ids as the \ + terms they replace", + ); } expr } diff --git a/src/interpreter/operator_conversion.rs b/src/interpreter/operator_conversion.rs index f347dd66..2c09be74 100644 --- a/src/interpreter/operator_conversion.rs +++ b/src/interpreter/operator_conversion.rs @@ -1462,7 +1462,7 @@ fn build_transact_store( /// The reply taps on a writer body's `` {`commit{writes, to_*} | `abort} `` /// decision — every field of the (dense) `commit` payload record other than /// `writes`, with its per-commit value type. A tap is a reply (`out << e`) that -/// desugar folded onto the writer body; for a commit store, op-conversion commits +/// channelize folded onto the writer body; for a commit store, op-conversion commits /// each tap as a write-only key so the reply rides the transaction's commit and is /// read back as a value-stream. Empty for a writer with no reply. fn body_tap_fields(body_ty: &Type) -> Vec<(String, Type)> { diff --git a/tests/compilation_pipeline/feeds_cases.rs b/tests/compilation_pipeline/feeds_cases.rs index 9025389c..89ef0efc 100644 --- a/tests/compilation_pipeline/feeds_cases.rs +++ b/tests/compilation_pipeline/feeds_cases.rs @@ -60,7 +60,7 @@ y <<= x y"#, make_int_list(&[0, 1]))] // Cross-cluster defer reference: `y` and `x` are separated by an // intervening non-Defer `let some_var = 5`, and `y` depends on `x` -// (via define). The desugar pass must topologically order the defers +// (via define). The channelize pass must topologically order the defers // across the intervening let so `x` is bound before `y`. #[case( r#"x = defer() @@ -412,7 +412,7 @@ fn scalar_define_into_defer_is_rejected() { /// Type errors in defer programs are reported against the *user's* program /// shape: inference now runs before `channelize`, so the rendered -/// message must not leak desugar artifacts (floated parameters, `to_` +/// message must not leak channelize artifacts (floated parameters, `to_` /// record fields, channel unions, scope-out bindings). #[rstest] #[timeout(Duration::from_secs(1))] @@ -442,7 +442,7 @@ x"#; ] { assert!( !rendered.contains(artifact), - "desugar artifact `{artifact}` leaked into a user-facing type error:\n{rendered}" + "channelize artifact `{artifact}` leaked into a user-facing type error:\n{rendered}" ); } } diff --git a/tests/compilation_pipeline/mutability.rs b/tests/compilation_pipeline/mutability.rs index 26a51b79..bc5b2843 100644 --- a/tests/compilation_pipeline/mutability.rs +++ b/tests/compilation_pipeline/mutability.rs @@ -922,7 +922,7 @@ fn an_equal_width_mut_parameter_still_accepts_a_mut_var() { /// /// Before this was reported here it tripped `check_scope_valid`, a debug-only /// regression net documented as never firing on a well-typed program — so a *release* -/// build had no check at all and reached the pre-desugar wall with a surviving mutable +/// build had no check at all and reached the pre-channelize wall with a surviving mutable /// type. #[rstest] #[case::direct(indoc! {r#" diff --git a/tests/inference_variants.rs b/tests/inference_variants.rs index ed4227b6..5d626a8f 100644 --- a/tests/inference_variants.rs +++ b/tests/inference_variants.rs @@ -722,7 +722,7 @@ fn a_bounded_variant_annotation_does_not_become_the_binders_type() { /// program that type-checks. /// /// So the gate asks the polarity-correct walk alone (`value_reaches` in -/// `src/ccl/infer/solve.rs`). Asserted through `check_pre_desugar` because that +/// `src/ccl/infer/solve.rs`). Asserted through `check_pre_channelize` because that /// wall is where the disagreement surfaces — inference itself reports `Int` and /// no error. #[test] @@ -748,6 +748,6 @@ fn an_upper_bound_alone_does_not_count_as_a_value_reaching_a_payload() { infer(&mut e, &mut ctx).expect("two call sites type-check"), int() ); - cambra::ccl::infer::check_pre_desugar(&e) + cambra::ccl::infer::check_pre_channelize(&e) .expect("every arm's recorded type is the join the wall recomputes"); } diff --git a/tests/type_check.rs b/tests/type_check.rs index bc0e5391..cd19bbe5 100644 --- a/tests/type_check.rs +++ b/tests/type_check.rs @@ -18,7 +18,7 @@ use cambra::ccl::{ FieldKey, HistoryKind, Lit, PredicateId, Type, ccl_utils::walk_refined_predicates, infer::{ - InferError, LocatedInferError, TypeInferenceContext, check_pre_desugar, infer, + InferError, LocatedInferError, TypeInferenceContext, check_pre_channelize, infer, lit_singleton, }, lower::{LoweringContext, lower_stmts}, @@ -1954,7 +1954,7 @@ fn infer_and_check(code: &str) -> Type { .into_result() .expect("lowering failed"); let ty = infer(&mut expr, &mut ictx).expect("inference failed"); - check_pre_desugar(&expr) + check_pre_channelize(&expr) .expect("post-inference consistency wall must accept the inferred tree"); ty } @@ -2140,7 +2140,7 @@ fn test_fed_defer_reads_through_aggregate() { fn test_defer_chain_flattens_feeds() { // `x <<= y` sets x's channel to y's whole stream. A feed reads through as // its stream, so x gets y's stream directly (a single feed layer, not - // nested); desugar later binds x to y's channel. + // nested); channelize later binds x to y's channel. let ty = infer_program("x = defer()\ny = defer()\nx <<= y\ny <<= [0, 1]\nx"); assert_eq!(*feed_value(&ty), int()); }