diff --git a/.github/scripts/shared-state/check_shared_state.py b/.github/scripts/shared-state/check_shared_state.py index 986aadf7a..57879d6e8 100644 --- a/.github/scripts/shared-state/check_shared_state.py +++ b/.github/scripts/shared-state/check_shared_state.py @@ -86,6 +86,7 @@ ("src/interpreter/mod.rs", "shared cell of `C`"), ("src/interpreter/tile_operators/cycle_slot.rs", "shared cell of `Option>`"), ("src/interpreter/tile_operators/fanout.rs", "cell of `bool`"), + ("src/interpreter/tile_operators/fanout.rs", "shared cell of `usize`"), ("src/interpreter/tile_operators/fanout.rs", "shared cell of `Box`"), ("src/interpreter/tile_operators/fanout.rs", "shared cell of `Box`"), ("src/interpreter/tile_operators/mod.rs", "ambient mutable state"), diff --git a/README.md b/README.md index f0180546e..9953a9322 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,24 @@ cargo run -- --inspect tests/programs/inner_join/program.cambra The inspector defaults to port 8080 (`--inspect=9090` to change it). After the program finishes, the process stays alive so you can browse `http://localhost:`; Ctrl+C to exit. +### Control port + +Pass `--control` to let a running program be diffed against, and replaced by, a new version of its source. Both endpoints take the new source as the query string or the request body: + +```bash +cargo run -- --control tests/programs/http_greeter/program.cambra + +# in another terminal, with the edited program in v2.cambra: +curl --data-binary @v2.cambra localhost:8081/diff +curl --data-binary @v2.cambra localhost:8081/update +``` + +`/diff` reports how the two versions differ and changes nothing. `/update` replaces the program in place: sockets stay open, every binding whose computation is unchanged keeps running, and a variable whose logic you did edit resumes from the value it was holding. Add `phase=&` before the source to diff somewhere other than the default (`lowered`, `inferred`, `inlined`, `channelized`, `lambda-elim`, `planned`). + +An update may change the program's logic freely, add endpoints, and stop serving ones it no longer wants (their addresses then answer 404). A variable may even move to another loop or into a transaction: it keeps its value and starts counting positions in whatever it now iterates. The one thing an update may not do is break continuity of state: a variable the running program is holding a value for must be one the new version declares, at the same type, or the update is refused and the running program is left serving. See [live-update.md](src/ccl/design/live-update.md). + +The control port defaults to 8081 (`--control=9090` to change it). + ## License Apache 2.0 — see [LICENSE](LICENSE). diff --git a/docs/design.md b/docs/design.md index 93542c32c..2bef76600 100644 --- a/docs/design.md +++ b/docs/design.md @@ -203,6 +203,7 @@ The connections between the layers above and the capabilities Cambra claims: - **Incremental views by construction.** Monotone tilings mean live aggregates are maintained, not recomputed. The materializable time-pinned view is the decided form of this — the history substrate is implemented, the transaction-handle read it needs is not yet (the ledger's `txn_kv` pins it). - **Verification [Sketched].** A small referentially-transparent core plus a refinement-typed checker means a semantic predicate established at one point composes across the whole program. The machinery exists today; whole-application contracts on top of it are the driving direction. - **Validation and program branching [Open].** Referential transparency makes program versions *syntactically comparable with well-defined semantics*, and temporal functional mutation makes state a value over time domains — *branchable and pinnable by construction*. Together they are the substrate for branching a running application — logic and state — exercising the branch under a realistic workload, and diffing behaviour. +- **Live update [Partial].** A running program can be replaced by a new version of its source over the control port (`--control`): `/diff` reports how the two versions differ, at a pipeline phase the caller picks, and `/update` swaps the program. The new version inherits the running one's sources and sinks, may add to them, and retires a route it stops serving, and inherits the operator behind every `Let` binding whose computation is unchanged; a variable whose logic the edit did touch resumes from the value it held. A variable that moves to another loop, or into or out of a transaction, keeps its value and counts positions in whatever it now iterates. The one refused update is one that cannot take over the state — a variable the new version no longer declares, or declares at a different type. See [live-update.md](/src/ccl/design/live-update.md). Running two versions at once is not implemented. - **Observability.** The compilation pipeline preserves a legible chain from source to running state; the web inspector (`--inspect`) serves the CHL AST, the lowered CCL, the operator graph, and live per-producer runtime state for any running program. ## Feature status at a glance @@ -221,6 +222,7 @@ The connections between the layers above and the capabilities Cambra claims: | `rec` fixpoint bindings | **[Decided]** | | Collections-as-functions model | Organizing idea decided; encodings **[Sketched]** | | `match` / `case` | Tag dispatch implemented; deeper patterns **[Tentative]** | +| Live update of a running program (`--control`) | Implemented for logic, endpoint changes, and state resume; running two versions at once is **[Open]** | | `while`, floats, imports, classes, exceptions | Absent (see the spec) | The [spec](chl-spec.md) carries the authoritative per-construct markers; [demo-programs.md](demo-programs.md) maps them to runnable programs and their blockers. diff --git a/src/ccl/ccl_utils.rs b/src/ccl/ccl_utils.rs index 0c38052f7..a6aedf67c 100644 --- a/src/ccl/ccl_utils.rs +++ b/src/ccl/ccl_utils.rs @@ -1178,6 +1178,55 @@ pub fn free_names_in_value(expr: &Expr) -> HashSet { out } +/// Every name free in `expr`, in its term structure **and** inside the +/// refinement predicates riding its type slots — the set counterpart of +/// [`is_free`], as [`free_names_in_value`] is of [`is_free_in_value`]. +/// +/// Answers "which bindings does this term read?" for a caller that must decide +/// something for all of them at once and would otherwise walk the term once per +/// candidate name. Predicates count because a term can depend on a binding +/// through one: a refinement is a term in a type position, and a domain +/// restriction built from it reaches the operator graph. +/// +/// Self-referential predicates terminate on the same `visited` discipline as +/// [`count_free`], and the refinement element binder is excluded in type +/// position for the reason given on [`count_free_in_type_with_visited`]. +pub fn free_names(expr: &Expr) -> HashSet { + fn go( + e: &Expr, + bound: &mut Vec, + visited: &mut HashSet, + out: &mut HashSet, + ) { + e.walk_type_slots(|ty| { + walk_refined_predicates(ty, visited, &mut |pred, vis| { + // A binder's declared type sits in the enclosing scope, so the + // predicate is walked under the binders in force *here*. + let mut inner = HashSet::new(); + go(pred, bound, vis, &mut inner); + out.extend(inner.into_iter().filter(|n| !n.is_elem())); + }); + }); + for_each_scoped_item(e, &mut |item| match item { + ScopedItem::VarRef(n) => { + if !bound.contains(n) { + out.insert(n.clone()); + } + } + ScopedItem::KeyRef(_) => {} + ScopedItem::Child { expr, binders } => { + let depth = bound.len(); + bound.extend(binders.iter().map(|b| b.name.clone())); + go(expr, bound, visited, out); + bound.truncate(depth); + } + }); + } + let mut out = HashSet::new(); + go(expr, &mut Vec::new(), &mut HashSet::new(), &mut out); + out +} + /// Value-only worker for [`is_free_in_value`]: the same fold over /// [`crate::ccl::scope::for_each_scoped_item`] as [`count_free`], minus the type /// slots (so a refinement on a `Lambda` param — which lives in the type — is diff --git a/src/ccl/context.rs b/src/ccl/context.rs index 6946fa305..07aa5e59d 100644 --- a/src/ccl/context.rs +++ b/src/ccl/context.rs @@ -19,7 +19,7 @@ use crate::{ check_pre_channelize, infer, typecheck, }, inline, lambda_elim, - lower::{LoweringContext, LoweringError, lower_stmts}, + lower::{LoweredRoute, LoweringContext, LoweringError, lower_stmts}, mut_elim, panes::gate_leaks, planning, @@ -32,6 +32,7 @@ use crate::{ }, interpreter::{ Consumer, DataSink, DataSourceDomainExtentImpl, Scheduler, StdinDataSource, + http_server::SharedHttpServer, operator_conversion::{ ConversionError, OpConversionContext, convert_record_fields_to_operators, convert_to_operators, @@ -356,6 +357,148 @@ impl CompileResultExt for Result> { } } +/// The live-update surface a caller of [`GlobalContext::reuse`] and +/// [`GlobalContext::state_conflicts`] reads their answers as. Both are produced +/// by operator conversion and returned from here, so this is the path a consumer +/// of the compilation API imports them by. +pub use crate::interpreter::operator_conversion::{ReuseTally, StateConflict}; + +/// One open `http_serve` route: what lowering binds it by, plus the listener +/// needed to stop serving it. +struct HttpRoute { + route: LoweredRoute, + server: Arc, +} + +/// The data sources and sinks a program holds open, and the HTTP listeners +/// behind them. +/// +/// Program state rather than compilation state. A listener's socket, its +/// routing-table entry, and the requests buffered behind it outlive the version +/// of the program that opened them, so they are what a replacement version +/// inherits: every compilation seeds a fresh [`LoweringContext`] from here, and +/// a `http_serve` naming a route already held binds it rather than opening a +/// second listener on the same address. +#[derive(Default)] +pub struct SourceSinkRegistry { + /// Every open source, by the name `Source(name)` resolves against. + sources: HashMap>>, + /// Every open `http_serve` route, by the route's source name. Keyed by route + /// rather than by response-binding name, which a new version may spell + /// differently. + http_routes: HashMap, + /// One listener per bound TCP port. + shared_servers: HashMap>, +} + +impl SourceSinkRegistry { + /// The names of every open source. + pub fn source_names(&self) -> impl Iterator { + self.sources.keys().map(String::as_str) + } + + /// Fold everything a completed lowering pass opened into the registry. + /// + /// The listeners are taken rather than copied, so that after this the + /// registry is the only long-term owner of a port and + /// [`release_unrouted_ports`](Self::release_unrouted_ports) dropping one + /// really closes it. + fn absorb(&mut self, lowering: &mut LoweringContext) { + self.sources.extend( + lowering + .registered_sources() + .map(|(n, s)| (n.to_string(), s.clone())), + ); + self.shared_servers.extend(lowering.take_servers()); + for (name, route) in lowering.registered_routes() { + let Some(server) = self.shared_servers.get(&route.port) else { + continue; + }; + self.http_routes.insert( + name.to_string(), + HttpRoute { + route: route.clone(), + server: server.clone(), + }, + ); + } + } + + /// Stop serving every route `still_bound` does not name. + /// + /// A route is registry state, so a version that stops binding one leaves it + /// dispatching into a source nobody reads: the request still matches, is + /// still buffered, and the client waits on a reply that will never be + /// computed. Retiring it makes the address answer 404 instead, which is what + /// "this program no longer serves that" should look like from outside. + fn retire_routes_absent_from(&mut self, still_bound: &HashSet) { + let dropped: Vec = self + .http_routes + .keys() + .filter(|name| !still_bound.contains(*name)) + .cloned() + .collect(); + for name in dropped { + let HttpRoute { route, server } = self.http_routes.remove(&name).expect("just listed"); + debug!("retiring route {} {}", route.method, route.path); + server.unregister(&route.method, &route.path); + self.sources.remove(&name); + } + self.release_unrouted_ports(); + } + + /// Drop the listener on every port no route is registered on any more. + /// + /// A port is held for as long as some version binds a route there, and no + /// longer. The listener is what makes an unrouted address answer 404, so + /// while any route survives on the port its siblings' addresses keep + /// answering; once the last one goes there is nothing left to answer for and + /// the address stops existing instead. Dropping the handle ends the + /// dispatcher thread and closes the socket + /// ([`SharedHttpServer`](crate::interpreter::http_server::SharedHttpServer)), + /// so a long-lived program that moves its endpoints between ports does not + /// accumulate a listener per port it ever served. + /// + /// Nothing is mid-flight on a port with no routes: a request there matched no + /// route, so it was answered 404 rather than buffered for a reader. + fn release_unrouted_ports(&mut self) { + let routed: HashSet = self.http_routes.values().map(|r| r.route.port).collect(); + self.shared_servers.retain(|port, _| { + let keep = routed.contains(port); + if !keep { + debug!("releasing port {port}: no route is registered on it"); + } + keep + }); + } + /// A [`LoweringContext`] holding everything this registry does. + /// + /// Every compilation starts from one of these, so a `http_serve` naming a + /// route already open binds it and one naming anything else opens it. + fn seed_lowering_context(&self) -> LoweringContext { + let mut lowering = LoweringContext::default(); + lowering.adopt_sources_and_sinks( + self.sources.iter().map(|(n, s)| (n.clone(), s.clone())), + self.http_routes + .iter() + .map(|(n, r)| (n.clone(), r.route.clone())), + self.shared_servers.iter().map(|(p, s)| (*p, s.clone())), + ); + lowering + } + + /// Compile `code` through `phase` against these sources and sinks, without + /// touching the running program — see [`compile_to_in`] for why that is safe + /// to do while it is serving. + pub fn compile_to(&self, code: &str, phase: Phase) -> Result> { + let mut lowering = self.seed_lowering_context(); + // Answering a question must not change what the program serves, so a route + // this registry does not hold is named rather than opened. + lowering.inherit_endpoints_only(); + compile_to_in(lowering, code, phase) + } +} + /// Bundles the per-stage registries needed to thread externally-managed data /// sources through the full CCL pipeline (lowering → type inference → compilation). pub struct GlobalContext { @@ -367,6 +510,9 @@ pub struct GlobalContext { conversion: OpConversionContext, /// Scheduler for triggering notifications. scheduler: Scheduler, + /// The sources and sinks the program holds open, which outlive any one + /// version of it. + sources_and_sinks: SourceSinkRegistry, } impl GlobalContext { @@ -381,12 +527,71 @@ impl GlobalContext { inference: TypeInferenceContext::new(), conversion: OpConversionContext::new(), scheduler: Scheduler::new(), + sources_and_sinks: SourceSinkRegistry::default(), }; let stdin = Rc::new(RefCell::new(StdinDataSource::new())); result.register_source(stdin); result } + /// A context around an already-seeded lowering registry, with every other + /// registry fresh. + /// + /// Everything but `lowering` is scratch and is dropped with the context, so a + /// compile through this opens nothing, adopts no operator, and leaves no + /// route registered — see [`compile_to_in`]. + fn scratch(lowering: LoweringContext) -> Self { + Self { + lowering, + inference: TypeInferenceContext::new(), + conversion: OpConversionContext::new(), + scheduler: Scheduler::new(), + sources_and_sinks: SourceSinkRegistry::default(), + } + } + + /// The sources and sinks the program holds open — what a replacement version + /// binds against rather than reopening. + pub fn sources_and_sinks(&self) -> &SourceSinkRegistry { + &self.sources_and_sinks + } + + /// Retire the running version's conversion context and carry its operators + /// forward to the version replacing it. + /// + /// Call once the running operator graph has been dropped. Each operator a + /// `Let` binding produced is offered to the next compilation by the identity + /// of the term it computes, and the replaced version's subscriptions to it + /// are neutralized ([`OpConversionContext::into_inheritance`]). The source + /// consumers the scheduler holds go the same way: a source handle outlives a + /// version, the subscriptions against it do not. + /// + /// Scheduler registrations need no attention here: they are weak and owned + /// by the producers that made them, so dropping the graph prunes exactly the + /// ones whose producer went with it (see [`Scheduler::add_source_handle`]). + pub fn retire_version(&mut self) { + // Every source records what its current producers have collectively + // released, so the replacement's new producers start there rather than at + // the oldest value it retains. An adopted operator keeps the registration + // it already has, and an element nobody finished is still delivered. + for source in self.sources_and_sinks.sources.values() { + source.borrow_mut().carry_release_to_new_producers(); + } + let previous = std::mem::replace(&mut self.conversion, OpConversionContext::new()); + self.conversion.inherit(previous.into_inheritance()); + } + + /// Install a fresh [`LoweringContext`] seeded from the source/sink registry, + /// discarding the previous compilation's lowering state. + /// + /// A fresh context rather than a reused one because everything else + /// `LoweringContext` accumulates (synthetic-name counter, transactional + /// variables, mutable-parameter functions) is per-pass, and carrying it into + /// a second pass would let one version's declarations leak into the next. + fn seed_lowering(&mut self) { + self.lowering = self.sources_and_sinks.seed_lowering_context(); + } + /// Returns the context for lowering pub fn lowering_ctx(&mut self) -> &mut LoweringContext { &mut self.lowering @@ -397,6 +602,17 @@ impl GlobalContext { &mut self.inference } + /// Every variable the running program holds that `planned` cannot take over + /// — one it no longer declares, or declares at a different type. + pub fn state_conflicts(&self, planned: &Expr) -> Vec { + self.conversion.state_conflicts(planned) + } + + /// How much of the version it replaced the last compilation adopted. + pub fn reuse(&self) -> ReuseTally { + self.conversion.reuse() + } + /// Returns the context for operator conversion pub fn conversion_ctx(&mut self) -> &mut OpConversionContext { &mut self.conversion @@ -414,7 +630,8 @@ impl GlobalContext { /// (pre-registered and discovered) is registered in one uniform pass. pub fn register_source(&mut self, source: Rc>) { let name = source.borrow().get_id().to_string(); - self.lowering.register_source(name, source); + self.lowering.register_source(name.clone(), source.clone()); + self.sources_and_sinks.sources.insert(name, source); } } @@ -1306,7 +1523,12 @@ fn run_frontend( return Err(errors); } - // Drain sink bindings discovered during lowering before taking sources. + // Fold anything this pass opened into the source/sink registry before the + // per-compilation registries are drained, so the next version inherits it. + // A no-op for an `Inherited` pass, which opens nothing. + ctx.sources_and_sinks.absorb(&mut ctx.lowering); + + // Drain sink bindings before taking sources. let sink_bindings = ctx.lowering_ctx().take_sink_bindings(); // Drain the lowering log and fold it once, at the lowering→pipeline @@ -1705,7 +1927,25 @@ fn run_passes( /// `Phase` is a legal stop. Which ones answer which question — and which ones a /// diff should be taken at — is `src/ccl/design/diffing.md`, "Which phase to diff". pub fn compile_to(code: &str, phase: Phase) -> Result> { - let mut ctx = GlobalContext::new(); + compile_to_in(GlobalContext::new().lowering, code, phase) +} + +/// Compile `code` through `phase` against an already-open source/sink set, +/// without touching the running program. +/// +/// Runs the same [`run_frontend`] every other entry point runs, over a +/// [`GlobalContext::scratch`] whose lowering registry is seeded and whose other +/// registries are thrown away on return. So nothing here binds a port, registers +/// a route, or mutates the live compilation contexts, which is what makes it safe +/// to answer a diff query about a *running* program: the naive alternative — +/// [`compile_to`]'s fresh [`GlobalContext`] — would try to bind a port the +/// running program already holds and fail. +fn compile_to_in( + lowering: LoweringContext, + code: &str, + phase: Phase, +) -> Result> { + let mut ctx = GlobalContext::scratch(lowering); Ok(run_frontend(&mut ctx, code, phase, &[], false)?.expr) } @@ -1727,7 +1967,8 @@ pub fn compile_program( code: &str, main_consumer: Box, ) -> Result> { - // The frontend is [], shared with []: parse through + ctx.seed_lowering(); + // The frontend is [`run_frontend`], shared with [`compile_to`]: parse through // join planning, every check between, and the three panes the inspector // reads — each of which is a captured phase output. // @@ -1749,6 +1990,17 @@ pub fn compile_program( lowering_projection, table_session, } = run_frontend(ctx, code, Phase::Planning, &PANES, true)?; + // A route the registry holds and this version did not bind is one the version + // stopped serving. Retire it, or it keeps matching requests and buffering them + // for a reader that no longer exists. A no-op for a first compilation, whose + // pass bound every route the registry has. + // + // Here rather than in `run_frontend`, because retiring is part of *installing* + // a version: a program's listeners are shared with every scratch compile taken + // against them, so a compile that only answers a question — `compile_to` for a + // diff — must not act on a route the running program still serves. + let bound = ctx.lowering.routes_bound_this_pass().clone(); + ctx.sources_and_sinks.retire_routes_absent_from(&bound); // The frontend ran to , which is past every pane boundary. let mut pane = |phase: Phase| { panes @@ -1776,6 +2028,10 @@ pub fn compile_program( // tail of the `Let*` chain rather than a `Record`; we synthesise a single // `("main", op)` entry for them so the rest of the function operates // uniformly on `Vec<(name, op)>`. + // Assign every mutable variable its identity before anything is built from the + // tree, so a store is built under the identity `state_conflicts` checked this + // version against. Both conversion entries below need it. + ctx.conversion_ctx().set_var_paths(&join_planned); let per_field_ops = if sink_bindings_registry.is_empty() { let op = convert_to_operators(&join_planned, ctx.conversion_ctx()).errs()?; vec![("main".to_string(), op)] @@ -1837,6 +2093,15 @@ pub fn compile_program( ) ); *producer_slot.borrow_mut() = Some(sink_producer); + // Kick the sink now it has something to pull. Operators notify from + // inside `subscribe` (an induction store does, to start its loop), and + // a `SinkConsumer` whose slot is still empty drops those — so the work + // already available when a version is installed needs a notification + // of its own. A first compile is carried by the source reporting its + // data as new; a *replacement* is not, because the version it replaces + // has already taken that report, so without this an update lands with + // unfinished work and nothing pulls it until the next arrival. + consumer_rc.borrow_mut().notify(); outputs.push(CompiledOutput { name, op, diff --git a/src/ccl/design/README.md b/src/ccl/design/README.md index b0b243185..90f35cd9e 100644 --- a/src/ccl/design/README.md +++ b/src/ccl/design/README.md @@ -31,6 +31,7 @@ CHL source | [optimization.md](optimization.md) | The optimization/compilation passes: inlining, lambda elimination, join/aggregate planning, algebraic simplification, and conversion to tile operators. | | [provenance.md](provenance.md) | How a node keeps its link to the source the user wrote across the whole pipeline: the `NodeId`/`Phase` identity primitives, the `ProvenanceTable` model and its fold, the recorder, the always-on lowering projection release diagnostics read, and what the inspector consumes. | | [diffing.md](diffing.md) | Program diffing: α-invariant content addressing of CCL terms and the GumTree correspondence between two compiled programs. | +| [live-update.md](live-update.md) | Replacing a running program with a new version: the endpoint set a version inherits, operator reuse keyed by the term a `Let` binds, and the weak subscriptions that let one graph be swapped for another. | Provenance is the one cross-cutting concern in the table: every pass above both preserves node identity and records what it rewrote, so diff --git a/src/ccl/design/diffing.md b/src/ccl/design/diffing.md index e4f64ec34..3969f71cf 100644 --- a/src/ccl/design/diffing.md +++ b/src/ccl/design/diffing.md @@ -29,7 +29,7 @@ What holds of the output, and the test that holds it: | | Held by | | --- | --- | -| Two compilations of one source diff as identical, at every phase | `every_stage_diffs_identical_source_as_identical` | +| Two compilations of one source diff as identical, at every phase | `every_phase_diffs_identical_source_as_identical` | | Identical trees have no divergences, and one shared root: the program | `identical_programs_have_no_divergences` | | An identity that varies between compilations (a `Name` uid) never reaches the result | `diff_is_robust_to_uid_nondeterminism` | | Renaming a binding is not a change | `renaming_a_binding_is_not_a_change` | @@ -279,7 +279,7 @@ Loop planning is where this bites. The mutable variable record a `Transact` denotes is typed with `Name::field_key()` labels, and folding the binder uid in would type the node `{acc#9: ([0, 2] ⇒ Int)}` in one compilation and `{acc#19: …}` in the next, with `.acc#9` against `.acc#19` reading it — two compilations -of the same source diffing as different, and every stage from planning down +of the same source diffing as different, and every phase from planning down unusable. The label has no need of a uid. It must be distinct only among the keys of one @@ -287,11 +287,11 @@ mutable variable record: every consumer resolves it against a `keys_map` built per `Transact` node, so accumulators in sibling loops live in different records and cannot collide. `field_key` is therefore the plain spelling. That leaves the distinctness as a property of spellings rather than of construction — a key is -the user's own variable name, distinct within its block; a label planning mints -indexed by position (`acc0`, `acc1`); or a writer's reply tap (`to__`), -which shares the record with both — so each site that builds a record from these -labels asserts it in debug (`hist_record` in `planning/loops.rs`, the two -`keys_map` inserts in `interpreter/operator_conversion.rs`). +the user's own variable name, distinct within the block or loop that declares +it, or a writer's reply tap (`to__`), which shares the record with them +— so each site that builds a record from these labels asserts it in debug +(`hist_record` in `planning/loops.rs`, the two `keys_map` inserts in +`interpreter/operator_conversion.rs`). The general rule this leaves: **a name rendered into a string is an identity the hash cannot normalize**, so a pass that needs a label derives it from something @@ -523,7 +523,7 @@ a b = sum([i * 2 for i in [1,2,3]]) a + b ``` -Diffed at the lowered stage they render as: +Diffed at the lowered phase they render as: ```text 2 shared · 1 changed · 1 moved · 0 deleted · 16 new @@ -572,7 +572,7 @@ shown with the reused `a` under it, because claiming `a` changed would be false. ### Worked example: one literal, and the duplicates around it A changed literal with the guard threshold edited to `1`, diffed at the lowered -stage: +phase: ``` # v1 # v2 @@ -660,15 +660,15 @@ are reachable at all — a caller debugging a phase wants them. ## How much to normalize -Two questions want two different stages, and neither one dominates. +Two questions want two different phases, and neither one dominates. **What runs together** is a question about the operator graph, so it is asked at -`Planning` — the shape operator conversion consumes, and therefore the only stage +`Planning` — the shape operator conversion consumes, and therefore the only phase where a claim about sharing compute is a claim about what executes. Anything a version guard or a shared store is derived from is read there. **Which source edit is this** is a question about the program the user wrote, so -it is asked at the earliest stage that can see the edit at all. Every pass +it is asked at the earliest phase that can see the edit at all. Every pass between the two rewrites the user's shape, and a rewrite spreads one edit over more of the tree: at `Planning` a comprehension's threshold change is four sites rather than one, and an accumulator's body change carries fourteen new nodes @@ -678,7 +678,7 @@ source. Two programs can also be the same computation written differently. The more of that the diff sees through, the more the two versions share — and the less -localized the answer becomes when they genuinely differ. Choosing a stage is +localized the answer becomes when they genuinely differ. Choosing a phase is choosing where on that curve to sit; the compiler's own passes do the work, so there is no separate rewriting system to keep honest. @@ -701,7 +701,7 @@ be reproduced and a drift in them is visible. Inlining is a trade, not an improvement: it makes moving code across a function boundary invisible, and in exchange reports an edit inside a shared helper once per call site, because the body it changed now appears once per call site. It is -the stage to pick when refactoring across function boundaries is the noise to +the phase to pick when refactoring across function boundaries is the noise to remove. ### `Infer` has already spent some of that locality @@ -721,7 +721,7 @@ the helper; `g(a)` and `g(b)` for two computed `Int`s key together and do not. Inlining then costs one further duplication per call site on top of whatever monomorphization already did — 1 → 2 in the shared row, 2 → 3 in the split one. -`Infer` is therefore the earliest stage this differ offers, not a stage that +`Infer` is therefore the earliest phase this differ offers, not a phase that has normalized nothing: the curve starts before its column. Below `Inline` the trade continues — every pass that rewrites the user's shape @@ -755,7 +755,7 @@ predicate is a shared `Rc` by design (`src/ccl/ty.rs`), so the mentions are identifiable — and it is not done here. See [Open threads](#open-threads). -The rule this leaves: diff at the earliest stage that can see the edit, unless +The rule this leaves: diff at the earliest phase that can see the edit, unless the answer is about the graph that runs, in which case diff at `Planning`. Reaching past `Inline` is not a better diff of the source; it is a diff of a different object. @@ -788,11 +788,11 @@ equivalence class and costs locality the same way inlining does. Nothing observed yet asks for them; the roadmap is driven by missed sharing that shows up in practice, not by completeness. -### The one that needs more than a stage +### The one that needs more than a phase Reordering two independent bindings is a true no-op — CCL's `let` is non-recursive and pure, so the operator graph depends on the dependency DAG, not -on the written order — and no stage fixes it, because the nesting is the order. +on the written order — and no phase fixes it, because the nesting is the order. See "Open threads". --- diff --git a/src/ccl/design/live-update.md b/src/ccl/design/live-update.md new file mode 100644 index 000000000..47bcc2770 --- /dev/null +++ b/src/ccl/design/live-update.md @@ -0,0 +1,467 @@ +# Live update: replacing a running program + +A running Cambra program can be replaced by a new version of its source without +restarting. The new version inherits the running one's external endpoints and +whichever of its operators compute the same thing; everything else is rebuilt. + +Two questions decide the design, and they have separate answers: + +- **What may change.** Its logic freely, and its endpoints by addition. +- **What may not.** State: a variable the running program holds a value for must + be one the new version declares, at the same type. +- **What survives.** Every `Let` binding and every `Transact` store whose + computation is unchanged, and every variable's value whether or not its store + was rebuilt. + +The entry point is `LiveProgram::update` in `src/live_program.rs`. Diffing the +two versions is [diffing.md](diffing.md); this doc covers what an update does +with the answer. + +## The model: every carrier has its own cut + +An update cuts the running program and hands whatever must survive across the cut. No cut is +program-wide. There is no instant the whole graph stops at, and two carriers' cuts need not fall at +the same input. + +A cut is a position in a sequence both versions count in. A source's stream is such a sequence: the +source outlives the version and hands the same positions to whoever reads it next. So is a +collection a version computes, named by the term that computes it — `[0, 2]` is the extent of +`["y", "z"]` and of `["p", "q"]` alike, so the extent alone does not say whether two versions are +counting the same elements. A commit clock is neither shared nor named: it is private and +restartable, nothing outside the store holds a tick, and a rebuilt store starts its clock at `0` and +loses nothing. A count of the columns a tile currently offers is not a sequence at all — it names a +position in a view whose contents depend on what has been released, so it means nothing to anyone +who did not emit it. + +Carriers are of two kinds, and each takes its cut from a different place. + +**A carrier that holds nothing takes its cut from its input.** An element-wise map, a feed: the +replacement recomputes from wherever its input still has work, and the release the retired +subscribers agreed on is what says where that is. This asks one thing of every producer — that it +release what it has finished, and only that. The agreement is what an input hands a subscriber that +registers after it: a `FanOut` seeds a new subscriber's guard from what it has already released +upstream, and every source seeds a newly-registered producer the same way — the trait method has no +default, and `ProducerReleases` is the per-producer bookkeeping each source keeps for it. An +operator whose subscribers +released it in full is withheld from the next version altogether — it can only answer empty, so +adopting it would bind a name to nothing (`FanOut::released_in_full`). + +**A carrier that holds a value takes its cut from itself.** An induction store's value at position +𝑝 already summarizes every position below 𝑝, so the store's own frontier says where the replacement +starts. Its source's release state does not and cannot. The value and the position travel together +in one `CarriedState`, because either alone decides a position twice or skips it. The position +travels with the sequence it counts in (`Sequence`), because a replacement counting in another one +has to start its own count. + +The two are not interchangeable in either direction. A carrier of the second kind reading its cut +off its source re-decides positions its recurrence already decided: a drive holds the input it reads +one position back through, so a source legitimately still owes elements the store has consumed. A +carrier of the first kind reading its cut off the end of the buffer drops everything that arrived +and went unhandled — over HTTP, a request accepted and then never answered. + +A carrier that holds progress and releases nothing has neither cut and replays its whole input. +`TransactDriver` therefore names the item it is attempting by absolute source position and releases +each item as it finishes, which makes it a carrier of the first kind: it holds no progress its +source does not, so an update has nothing to hand over. +`a_transaction_writer_does_not_replay_what_it_committed` pins that, and the same release bounds a +transactional source's buffer while the program runs. + +Every carrier in the program, and where its cut comes from: + +| Carrier | Cut | Mechanism | +| --- | --- | --- | +| A `Let` binding or store the new version also computes | None — nothing is replaced | `resolved_hash` match, one more `FanOut` branch | +| An element-wise map or a feed over a shared operator | Its input's agreed release | A new subscriber's guard starts at what the `FanOut` has released | +| The same, over a source | Its source's agreed release | `carry_release_to_new_producers` | +| An induction store counting in the sequence its predecessor counted in | Its predecessor's frontier | `CarriedState`, each variable's value and position together | +| Any other induction store | Where its source will next offer a producer, or `0` for a collection | `first_position_for_a_new_producer` | +| A transaction writer's item cursor | Its source's agreed release | Absolute item positions, released on the commit-ack | +| A commit clock | None — private and restartable | A rebuilt store seeds at `0` | +| A route, its listener, and the requests behind it | None while any version still binds a route on the port | `SourceSinkRegistry` | + +## Sources and sinks outlive a version + +A `SourceSinkRegistry` (`ccl/context.rs`) holds a program's open data sources, the +reply sink of each open `http_serve` route, and the listener behind each bound +port. A listener's socket, its routing-table entry, and the requests buffered +behind it are program state rather than compilation state: they outlive the +version of the program that opened them. + +Compiling any version seeds a fresh `LoweringContext` from that registry. An +`http_serve` call naming a route the registry holds binds it — keeping the +listener and everything buffered behind it — and one naming a route it does not +hold opens it, in a replacement exactly as in a first version. Routes are keyed +by their source name rather than by the response binding's name, which a new +version may spell differently. + +Seeding the registry is also what lets a *diff* be answered about a running +program at all: compiling the new version in a fresh `GlobalContext` would try to +bind a port the running program holds and fail. + +## The one guard: a version must be able to take over the state + +`LiveProgram::update` compares the variables the running program holds against +those the new version declares, read off its planned tree +(`OpConversionContext::state_conflicts`). Two things are refused: + +- **A variable the new version no longer declares.** Its value has nowhere to be + seeded and would be discarded. +- **A variable it declares at a different type.** Its value cannot seed a store + built for another shape. Allowed through, the store is constructed around a + constant of the wrong extent and the *process* dies on the next pull + (`Scalar(Strings([…])) vs Scalar(Int)`), taking every endpoint with it — which + is why this is checked rather than left to fail later. + +The check runs before anything is torn down, so a refused update leaves the +program whole. + +Nothing else is refused. An earlier version of this pass froze the source and +sink set and rejected any new `http_serve`; adding one turns out to work — the +added route serves as soon as the swap completes, and what was already there +keeps its state — so the restriction bought nothing and the guard now matches +what the mechanism does. + +Losing a value is refused because it is the one failure an author cannot +observe: the program carries on answering, and only the accumulated history is +gone. Every other change either works or fails visibly. + +### A route a version stops serving is retired + +Removing an `http_serve` is accepted, and the route goes with it. A listener and +its routing-table entry are registry state, so they outlive the version that +opened them: left registered, the address would keep matching requests and +buffering them for a reader that no longer exists, and the client would wait on a +reply nobody computes. `SourceSinkRegistry::retire_routes_absent_from` compares +the routes the pass bound against those the registry holds and unregisters the +difference, so the address answers 404 — which is what "this program no longer +serves that" should look like from outside. + +Retiring belongs to installing a version, not to compiling one, and it runs in +`compile_program` rather than in `run_frontend` for that reason. A diff compiles +the new version against the running registry, so the listeners it sees are the +running program's; a compile that answered a question by unregistering a route +would make `/diff` change what the program serves. +`diffing_against_a_version_that_drops_a_route_does_not_retire_it` pins it. + +The port goes when its last route does. While a sibling route survives there the +listener stays, which is what makes the retired address answer 404 rather than +refuse a connection; once nothing is registered there is nothing left to answer +for, so `release_unrouted_ports` drops the listener and the socket closes. That +bounds what a long-lived program holds: a version that moves its endpoints to +another port releases the one it left, instead of the process keeping every port +it ever served. Dropping the handle is what closes it — `SharedHttpServer` holds +the listener alongside the dispatcher thread and unblocks the thread on drop, so +the thread's own shutdown path runs and its `Server` goes with it. +`a_port_whose_last_route_goes_is_released` covers both halves. + +## Operator identity is the term it computes + +Operator conversion consults the previous version at every `Let` binding +(`OpConversionContext::bind_let`). A binding is reused when the previous version +bound the same computation, and the fan-out behind it is branched instead of +rebuilt. + +A `Let` is the reuse boundary because it is already the sharing boundary. Every +binding compiles to `Rc::new(Memo::new(op))` so that several uses can +draw on one operator, and a new version's use is one more use. A late branch does +not re-subscribe upstream: it pulls the same `MemoProducer`, whose cache is +cumulative. That is what hands a reused operator's accumulation to the version +that inherits it. + +Identity is the `resolved_hash` of the bound term (`ccl/content_hash.rs`), taken +against the correspondents of the bindings in scope. It is α-invariant, so two +independent compilations of one source agree, and type-aware, so a refinement +change is a change. + +### Reuse is hereditary + +An operator is adopted only when every binding its term reads was adopted too, so +a carried-forward operator is never left reading a subgraph the update rebuilt. +`OpConversionContext::rebuilt` records the bindings this compilation built, and +`reads_only_adopted` declines any term with a free name among them +(`ccl_utils::free_names`, which counts occurrences inside refinement predicates +as well as in the term). Bindings are bound in dependency order, so the check is +transitive: a binding that reads a rebuilt one is itself recorded as rebuilt. + +Naming which bindings were rebuilt, rather than folding that into the +correspondent, is what keeps the correspondent stable. A binding's correspondent +is the identity hash of the term it computes, in every compilation and whether or +not the operator behind it was adopted. Two compilations of one source therefore +agree on it, and an unchanged part of a program is recognized on the first update. + +Encoding "was it rebuilt" in the correspondent instead makes reuse depend on how +many updates have happened: the correspondents a first compilation hands out are +then all values no later compilation reproduces, so every binding that reads +another is +rebuilt on the first update and reuse only settles from the second. +`reuse_does_not_depend_on_how_many_updates_came_before` pins that. + +### Stores are bindings too + +A program's mutable variables live in a `Transact` store bound to `__hist`, and +every read of one is a projection `__hist.k` off that binding, where `k` is the +variable's own spelling. The store is bound +by `OpConversionContext::bind_store` on the same terms as any other binding: it +carries a correspondent, it is keyed by the identity of its `Transact` term, and a +matching one is adopted whole. Adopting a store is what carries an accumulator +across an update, because the store is where the accumulation lives. + +Registering the store outside the conversion scope is what the correspondent +exists to +prevent. `__hist` would then be free in every term reading a mutable variable, and +`hash_free_var` identifies an unresolved free variable by its bare spelling — so +every such term would hash the same however the recurrence was edited, and its +operator would be reused against a store that no longer computed what it had. +`an_edit_to_the_accumulating_loop_takes_effect` pins that. + +One store covers one causal group, so an edit anywhere in a group rebuilds that +group's store; two independent mutable variables get two stores and are +independently reusable. + +### What is never reused + +A binding compiled under an iteration (`BindingKind::Aligned`) is rebuilt. Its +operator is parameterized by an iteration input threaded into it at conversion +time; the input is not part of the term, so the term does not identify the +operator. + +## A subscription lasts as long as its producer + +Replacing a graph while sharing operators with it requires knowing which +subscriptions are still real. Three registrations answer that the same way: the +subscriber owns its side, and the registry holds a weak reference. + +- **Fan-out branches.** `FanOutShared::subscribers` holds a weak reference per + slot whose strong side lives in the `FanOutProducer` that slot handed out. A slot + whose producer is gone is skipped when notifying and when intersecting release + guards. Skipping matters: the guards are intersected before anything is + released upstream, so a subscriber that will never release again would + otherwise pin the intersection where it stopped and the input would retain + everything from there on. A producer addresses its guard by slot number, so the + number lives in a `Cell` the producer and the registry share: `FanOut::reopen` + drops the dead slots and writes each survivor its new number. Without that + renumbering the slot list would grow by one dead entry per replaced subscriber + on every update and never shrink, and both the notify walk and the release + intersection scan it — + `reopening_a_fan_out_drops_dead_slots_and_renumbers_the_rest` pins that the + survivor keeps the guard it released. +- **Scheduler wake-ups.** `Scheduler::add_source_handle` records a + `Weak>`, and the `IterateExtentProducer` that registered + it owns the strong side. A source handle outlives a version; the subscriptions + against it do not. A strong registration would keep every operator any version + ever subscribed alive and being notified. +- **Sink dispatch.** `SinkConsumer::detach` clears the producer slot at teardown. + Dropping the compiled outputs does not end a replaced version's dispatch on its + own: an operator the next version carries forward still holds the notification + closure that reaches the old sink consumers, so they would keep being woken and + keep writing to sinks the new version now owns. Clearing the slot also drops + the operators behind it, which is what lets the fan-outs they subscribed to see + those subscriptions end. + +## Order of an update + +1. Render the difference between the running source and the new one, which + compiles both to `Phase::AsOfRead`. +2. Compile the new version to `Phase::Planning` against the endpoint registry and + run the state guard on the planned tree. Steps 1 and 2 open nothing and build + no operators, so a version that fails either leaves the running program + serving. +3. Tear down the running graph: detach its sinks, drop its outputs. +4. `GlobalContext::retire_version` moves the retiring conversion context's + operators into the next compilation's inheritance. +5. Compile and subscribe the new version against the same registry, which now + binds every endpoint the retired version left open, and opens the ones it adds. +6. Notify each sink, so whatever is already available is pulled. + +Step 6 is not redundant with the notifications `subscribe` raises. An operator +notifies from inside `subscribe` — an induction store does, to start its loop — +and a sink consumer's producer does not exist until `subscribe` returns, so those +notifications reach a consumer with nothing to pull and are dropped. A first +compile does not notice: a source holding data reports it as new on the next poll, +which drives everything. A replacement is not covered by that, because the version +it replaces already took the report. Without step 6 an update installed while work +is outstanding — a fold caught partway, a request accepted and not yet answered — +sits until the next arrival +(`a_version_installed_mid_fold_is_pulled_without_a_new_arrival`). + +Only step 5 opens anything. Steps 1 and 2 run with `Endpoints::Inherited`, so a +route the registry does not hold is named rather than opened: their contexts are +thrown away but a socket is not, and opening one would make asking a question +change what the program serves and leave step 5 unable to bind the port. + +Steps 3 and 4 come after step 2 so that a rejection is never destructive, and +before step 5 so that what the new version inherits is held by the inheritance +and not also by a graph still running. + +An accepted update therefore compiles four times: twice for the diff, once to +`Planning` for the guard, and once for real. `run_frontend` goes from source to a +stop phase and there is no way to continue a stopped tree into operator +conversion, so the guard's tree cannot be the one that gets built — which is why +`LiveProgram::update` documents a panic for the two compiles disagreeing. +Compilation is cheap next to the state the swap preserves, and the ordering is +what makes a rejection non-destructive, so the cost buys the guarantee. + +## What an update does not do + +- **Start a rebuilt store empty.** It resumes instead — see + [Rebuilding a store resumes it](#rebuilding-a-store-resumes-it). +- **Reuse across a change of shape.** Adoption is all-or-nothing per binding and + keyed by exact term identity, so a term that changed at all is rebuilt whole. + Nothing recognizes that an edited term still computes most of what it did. +- **Run both versions.** The replaced version is dropped. Running two versions + concurrently over shared state is a separate model. +- **Reuse below a `Let`.** Reuse is offered at binding boundaries only. Offering + it at an arbitrary node would mean wrapping every node in `Memo`/`FanOut`, + changing execution characteristics program-wide. + +## Rebuilding a store resumes it + +A rebuilt store does not restart its variables from the inits the source +declares. Each one resumes from the value the retired version left it holding, so +editing a loop changes what it does next without discarding what it had +accumulated. Editing how a guestbook formats an entry leaves the entries it +already recorded as they were and formats the next one the new way. + +Three pieces carry that: + +- **The value.** A store's value rides its own cyclic `FanOut` as a + `Tile::Store`, so `live_state` reads each carried key off + `FanOut::cached_tile` with `store_frontier` / `store_value_at`. Reading the + fan's own memo rather than keeping a copy beside the operator is what keeps + this off the shared-state ledger `./ci.sh shared_state` maintains: no value + crosses between operators outside a tile. +- **The name.** State is keyed by a `VarPath`: the variable's own spelling plus + its index among the variables of that spelling, in tree order. The spelling + carries the meaning — a writer's write set is keyed by the variable written, so + the name survives from the source text to the store — and the index only + disambiguates. Counted among the variables sharing the spelling rather than + among all of them, so a stateful loop added anywhere shifts nothing unless it + declares that same name. + + **A spelling is not unique, and nothing computed can stand in for one.** A + declaration can be shadowed, and a function holding a whole stateful loop + declares one variable per call site once inlining has cloned its body. Two such + instantiations can differ *only* in their writer bodies once arguments are + substituted, so a content-derived identity either fails to tell them apart or + changes under exactly the edit state has to survive. The index is what is left, + and it is why identity is assigned by one walk + (`OpConversionContext::set_var_paths`) whose answers both the guard and + conversion read, rather than derived twice. +- **The position.** A rebuilt store resumes at the retired store's frontier, and + each variable's seed is that store's value at the same position + (`CarriedState`). The value and the position travel together because either one + alone decides a position twice or skips it, and they hang off the *variable* + because a variable is what has an identity: a store has none, so two commit + stores at different frontiers would otherwise race to set one position. A + rebuilt store reads the position back off any variable it declares. Both the + store's seed tick (`CommitEngine::seeded_at`) and the drive's window base come + from `resume_at`, and they must agree — `InductionDriver` asserts that a + decision cannot precede the input it decides. + + A position only means something in the sequence it was counted in, so the two + are carried separately and a variable whose new version counts in another + sequence seeds its value and starts counting again. That is a variable moving + between loops, or into a transaction, or a program moving to another port: the + positions it is about to decide belong to something its predecessor never read, + so none is decided twice and none is skipped, while the value — which is the + variable's, not the sequence's — goes on. + + `Sequence` is what two versions compare. A source is named by itself, since it + outlives every version reading it; a collection is named by the identity of the + term that computes it, since the extent it iterates does not distinguish + `["y", "z"]` from `["p", "q"]` and resuming the second fold at the first's + frontier would skip elements nothing ever read. A transaction hands on no + position at all: its clock restarts with the store that counts it, and the + replacement seeds tick `0` from the carried value. + + **Starting again is not starting at `0`.** A source it moved to may have been + read all along by some other loop, which released what it consumed, and the + source will not offer those positions again. So a store with no predecessor over + its own domain starts at `first_position_for_a_new_producer` — `0` for a source + nothing has read, and the released frontier otherwise. Basing a drive below that + makes it wait for an element that is not coming, which is a silent stall rather + than a wrong answer. The same holds for a store the update *adds*: a loop that + gains an accumulator over a source the program was already reading has nothing + carried and still cannot start at `0` + (`a_stateless_loop_may_gain_an_accumulator_over_an_advanced_source`). + + A collection is a sequence like any other. A fold over one resumes at the + position it had reached, so an edit inside the loop governs the elements that are + left rather than replaying the ones already folded, and a fold over a *different* + collection starts that collection from its first element. The positions the + predecessor decided are not re-decided and are not re-read: the resumed store + seeds tick `0` with the value handed over, so a reader enumerating the whole + collection reads that value for them. A fold caught partway is where this is + visible — the elements below the cut keep what the retired version decided and + the rest are the new one's + (`a_fold_interrupted_partway_resumes_at_the_position_it_reached`). +- **What the source still owes.** A source hands a producer registering after the + swap the release state its retired producers agreed on, which runs *below* a + store's resume position rather than deciding it: a drive holds the input it + reads one position back through, so the source owes the replacement elements + the recurrence has already decided, and the drive holds those without + re-deciding them. See + [The model: every carrier has its own cut](#the-model-every-carrier-has-its-own-cut). + +### Positions are absolute, in the store and in the window alike + +`DriverWindow` addresses rows absolutely: `rows[i]` is position `base + i`, and +released rows compact off the front without renumbering, because the body looks a +decision up by domain value. A drive that starts with its source leaves `base` at +`0`, where row index and position coincide; a resuming drive sets it to the +position it starts from. + +Getting that wrong stalls the drive rather than misreading it — the decision +lookup finds no row at the absolute position and the drive stops without +advancing, so the resumed loop answers nothing while the rest of the program +keeps serving. `a_store_resumes_however_far_its_source_has_advanced` pins it, +driving six positions before the update; at one or two the two indexings overlap +enough to mask it. + +## What the guard lets through + +Everything that leaves the state takeable. Measured across the shapes an edit can +have: + +| Change | Outcome | +| --- | --- | +| Logic of a loop or a transaction writer | Accepted; the variable resumes | +| A loop gains an accumulator | Accepted; the others resume, the new one starts at its init | +| A variable's declared init changes, type unchanged | Accepted; the carried value wins, the init is only for a fresh start | +| A whole stateful loop is added | Accepted; existing state untouched, and the new store starts where its source has got to | +| A route is added | Accepted; it serves as soon as the swap completes | +| A route is removed | Accepted; the route is retired and answers 404 | +| A loop loses an accumulator | Refused, naming it | +| A variable's type changes, records included | Refused, naming both types | +| A variable moves to another loop, or to or from a transaction | Accepted; it seeds with the value it held and decides its new loop's positions from `0` | +| A loop reads another source, a port change say | Accepted; same as above, and the port it left is released | +| Two loops swap which source they read | Accepted; each keeps its value and continues on the source it moved to | +| A variable moves to a loop over a fixed collection | Accepted; same as above — the value seeds and the collection folds on top of it | +| The body of a loop over a fixed collection is edited | Accepted; the fold resumes at the position it had reached, so the new rule governs the elements left | +| The collection itself is edited | Accepted; the new collection counts in its own sequence, so it is folded whole | + +An update is atomic with respect to requests: under concurrent load every request +is answered by exactly one version, and the versions do not interleave. + +A program whose output is its `main` value rather than a sink updates the same +way. Such a program is not short-lived — `stdin` is unbounded, so the binary's +own driver loop keeps running and services the control port between pulls. Its +loops are identified by their source like any other, so the guard names one +plainly: `` `n`, of the loop over `stdin` ``. + +Its state is as live as a sink program's, and reads the same way: + +- A pure element-wise transformation splits exactly at the swap. Eight lines with + the swap after the fourth emit four under the old rule and four under the new, + each once: the stream is neither replayed through the new version nor dropped + at the handover. +- An accumulator carries across. Counting `+1` per line, switched to `+2` after + half the stream, the value at EOF is `1.5` times the line count — each half + counted by the rule that was in force when it arrived. +- Feeding the accumulator out (`out << n`) reports it per line rather than only + at EOF, so the swap is observable mid-stream: `1, 2, 4, 6` resumes from `2` + rather than restarting. + +What a value like a bare trailing `n` reports is decided by *when* it is read, +and reading it at the tail of the program means EOF — which is a property of that +program, not a limit on what an update can carry. diff --git a/src/ccl/diff.rs b/src/ccl/diff.rs index 852218d9a..31679a7e2 100644 --- a/src/ccl/diff.rs +++ b/src/ccl/diff.rs @@ -40,16 +40,16 @@ //! common. Those two are the actionable form: the first says where a version //! guard goes, the second says what the two versions can compute once. //! -//! # Stage-agnostic +//! # Phase-agnostic //! //! [`diff`] is a pure function of two [`TypedExpr`] trees and does not care -//! which pipeline stage produced them, so one implementation serves every +//! which pipeline phase produced them, so one implementation serves every //! [`Phase`]: the caller chooses how much of the compiler's own //! rewriting to diff through by choosing which trees to pass. Nothing in the -//! matcher is stage-specific — [`content_hash`] is uid-robust (free names by +//! matcher is phase-specific — [`content_hash`] is uid-robust (free names by //! spelling) and type-aware, which is what lets one core cover the lot, //! including the `LetRec` and `Transact` shapes that exist only below the -//! mutability phases. Which stage answers which question is +//! mutability phases. Which phase answers which question is //! `src/ccl/design/diffing.md`, "Which phase to diff". //! //! # Scope of this implementation @@ -398,7 +398,7 @@ fn anchor_roots(s: &Indexed, d: &Indexed, m: &mut Matching) { recover(s, d, ROOT, ROOT, m); } -/// Compile two source programs to `stage` and diff them — the end-to-end entry +/// Compile two source programs to `phase` and diff them — the end-to-end entry /// point from source. The classified [`Diff`] borrows the two compiled trees, /// which live only for the duration of this call, so the result is delivered to /// `f`; return out of it whatever you need to keep (e.g. counts, cloned nodes). @@ -410,11 +410,11 @@ fn anchor_roots(s: &Indexed, d: &Indexed, m: &mut Matching) { pub fn diff_programs( src: &str, dst: &str, - stage: Phase, + phase: Phase, f: impl FnOnce(&Diff) -> R, ) -> Result> { - let a = compile_to(src, stage)?; - let b = compile_to(dst, stage)?; + let a = compile_to(src, phase)?; + let b = compile_to(dst, phase)?; Ok(f(&diff(&a, &b))) } @@ -1816,21 +1816,21 @@ mod tests { /// Pre-uniquify CCL (`Raw` names, before inference), via the public API. fn lower(code: &str) -> TypedExpr { - compile_to(code, Phase::Lower).expect("compile to lowered stage should succeed") + compile_to(code, Phase::Lower).expect("compile to lowered phase should succeed") } /// Post-inference CCL (uniquified, fully typed), via the public API. fn lower_and_infer(code: &str) -> TypedExpr { - compile_to(code, Phase::Infer).expect("compile to inferred stage should succeed") + compile_to(code, Phase::Infer).expect("compile to inferred phase should succeed") } // Realistic CHL programs exercising records, list comprehensions, filters, // aggregates, projections, joins, def/yield generators, groupby, induction // accumulators, and transactional registers. Between them they cover every - // node kind that reaches these two stages: `Defer`/`Feed` (generators), + // node kind that reaches these two phases: `Defer`/`Feed` (generators), // `Case` (guards), `Cast` (comprehension filters), `For`/`MutWrite` // (accumulators), and `Begin` (transaction blocks). `LetRec` and `Transact` - // are born *below* the inferred stage — the mutability phases build them — + // are born *below* the inferred phase — the mutability phases build them — // so no source program can exercise them here; `content_hash` covers them // structurally instead. const FILTER_AGG: &str = indoc! {r#" @@ -2526,14 +2526,14 @@ mod tests { } #[test] - fn every_stage_diffs_identical_source_as_identical() { + fn every_phase_diffs_identical_source_as_identical() { // The property the whole analysis rests on, over the shapes the corpus // reaches: compiling one source twice — independent contexts, fresh // binder uids — must produce trees the differ cannot tell apart. A // failure means some pass has let a run-varying identity leak into the // hash, which would make every real diff untrustworthy. // - // It covers every stage because that is exactly how the leak was found: + // It covers every phase because that is exactly how the leak was found: // `Transact` labelled its mutable variable record with // `Name::field_key()`, which folded the binder uid into a `String`, and // once a name is a record label no amount of uid-robustness in the @@ -2555,7 +2555,7 @@ mod tests { ("transaction", TXN), ("source", "[\"> \" + line for line in stdin()]\n"), ]; - for stage in [ + for phase in [ Phase::Lower, Phase::Uniquify, Phase::Infer, @@ -2569,12 +2569,12 @@ mod tests { ] { for (label, src) in corpus { let (a, b) = ( - compile_to(src, stage).expect(label), - compile_to(src, stage).expect(label), + compile_to(src, phase).expect(label), + compile_to(src, phase).expect(label), ); assert!( diff(&a, &b).is_identical(), - "{label} at {stage:?} is not stable across compilations:\n{}", + "{label} at {phase:?} is not stable across compilations:\n{}", diff(&a, &b) ); } @@ -2785,15 +2785,15 @@ mod tests { prog("(await_final(a), await_final(b))"), prog("(await_final(b), await_final(a))"), ); - for stage in [Phase::Channelize, Phase::Planning] { + for phase in [Phase::Channelize, Phase::Planning] { let (a, b) = ( - compile_to(&v1, stage).expect("v1"), - compile_to(&v2, stage).expect("v2"), + compile_to(&v1, phase).expect("v1"), + compile_to(&v2, phase).expect("v2"), ); let r = diff(&a, &b); assert!( !r.is_identical(), - "a swapped pair of registers is a change at {stage:?}:\n{r}" + "a swapped pair of registers is a change at {phase:?}:\n{r}" ); // The swap is one site — the tuple — rather than the whole // recurrence: the two reads pair with their counterparts and move. @@ -2803,7 +2803,7 @@ mod tests { sites.as_slice(), [Divergence::Changed(m)] if matches!(m.dst.node, TypedExprNode::Tuple(_)) ), - "at {stage:?} the swap must localize to the tuple:\n{r}", + "at {phase:?} the swap must localize to the tuple:\n{r}", ); // And the tuple itself is not offered for reuse: its two elements // returned swapped values. @@ -2811,7 +2811,7 @@ mod tests { !r.shared_roots() .iter() .any(|m| matches!(m.src.node, TypedExprNode::Tuple(_))), - "at {stage:?} the swapped tuple is not reusable wholesale:\n{r}", + "at {phase:?} the swapped tuple is not reusable wholesale:\n{r}", ); } } @@ -2819,7 +2819,7 @@ mod tests { #[test] fn compile_to_rejects_what_compile_program_rejects() { // `compile_to` is a second path through the frontend, so a program the - // real pipeline refuses must not yield a stage snapshot: the differ + // real pipeline refuses must not yield a phase snapshot: the differ // would otherwise be handed a tree whose illegal construct was silently // dropped, and two versions differing only in it would diff as // identical. `x := 2` writes to an immutable binding, which @@ -2829,7 +2829,7 @@ mod tests { x := 2 x "}; - for stage in [ + for phase in [ Phase::Infer, Phase::Inline, Phase::Channelize, @@ -2837,8 +2837,8 @@ mod tests { Phase::Planning, ] { assert!( - compile_to(src, stage).is_err(), - "a write to an immutable binding must be rejected at {stage:?}", + compile_to(src, phase).is_err(), + "a write to an immutable binding must be rejected at {phase:?}", ); } // `Lowered` is below every check by construction — it is the tree as @@ -2860,9 +2860,9 @@ mod tests { let mut ctx = GlobalContext::new(); let compiled = compile_program(&mut ctx, src, Box::new(|| {})) .unwrap_or_else(|e| panic!("compile_program failed on {src:?}: {e:?}")); - let staged = compile_to(src, Phase::Planning) + let planned = compile_to(src, Phase::Planning) .unwrap_or_else(|e| panic!("compile_to failed on {src:?}: {e:?}")); - let d = diff(&compiled.ast, &staged); + let d = diff(&compiled.ast, &planned); assert!( d.is_identical(), "the two entry points disagree on {src:?}: {:?}", @@ -2888,18 +2888,18 @@ mod tests { #[test] fn diff_programs_end_to_end_from_source() { - // The public single-call entry: compile both sources to a stage and + // The public single-call entry: compile both sources to a phase and // diff, results delivered through the closure. // // A filter-threshold edit (`>= 18` → `>= 21`) is reflected at the - // lowered stage. + // lowered phase. let changed = diff_programs(FILTER_AGG, FILTER_AGG_21, Phase::Lower, |d| { d.updated().count() }) .expect("compile + diff should succeed"); assert!(changed > 0, "the edit is reflected"); - // Identical programs diff as identical at the inferred stage. + // Identical programs diff as identical at the inferred phase. let identical = diff_programs(FILTER_AGG, FILTER_AGG, Phase::Infer, |d| d.is_identical()) .expect("compile + diff should succeed"); assert!( @@ -2915,7 +2915,7 @@ mod tests { // the public API handles sources, not just literal programs. let prog = "[\"> \" + line for line in stdin()]\n"; let identical = diff_programs(prog, prog, Phase::Infer, |d| d.is_identical()) - .expect("stdin program should compile to the inferred stage and diff"); + .expect("stdin program should compile to the inferred phase and diff"); assert!(identical); } } diff --git a/src/ccl/infer/emit.rs b/src/ccl/infer/emit.rs index 16515d645..d67230ef9 100644 --- a/src/ccl/infer/emit.rs +++ b/src/ccl/infer/emit.rs @@ -2004,10 +2004,10 @@ fn emit_transact_writer( } let body_dom = accumulator_body_domain(snaps, item); - // Body codomain: `{commit: Bool, writes: Tuple(new_j…)}`, with `new_j` a - // fresh var bounded above by `write_keys[j]`'s value type. `writes` is a - // positional tuple built directly (not via `product`, whose empty case - // collapses to `Record([])`): even a single-key write set is `Tuple([_])`. + // Body codomain: `{commit: Bool, writes: {k_j: new_j…}}`, with `new_j` a + // fresh var bounded above by `write_keys[j]`'s value type. The write set is + // keyed by the variable written, so a slot carries which variable it belongs + // to all the way to the store — see `src/ccl/design/mutability.md`. let mut new_tys: Vec = Vec::with_capacity(writer.write_keys.len()); let mut news: Vec<(Type, Type)> = Vec::with_capacity(writer.write_keys.len()); for wk in &writer.write_keys { @@ -2030,7 +2030,14 @@ fn emit_transact_writer( let mut payload: BTreeMap = BTreeMap::new(); payload.insert( FieldKey::Name(SmolStr::from(crate::ccl::F_WRITES)), - Type::Tuple(new_tys), + Type::Record( + writer + .write_keys + .iter() + .map(|wk| wk.field_key()) + .zip(new_tys) + .collect(), + ), ); let decision_codom = Type::variant(vec![ (FieldKey::Name(SmolStr::from(V_COMMIT)), product(payload)), diff --git a/src/ccl/lower/mod.rs b/src/ccl/lower/mod.rs index bccd3ac37..9a3b6f550 100644 --- a/src/ccl/lower/mod.rs +++ b/src/ccl/lower/mod.rs @@ -88,7 +88,10 @@ use crate::{ Expr as ChlExpr, RecordField, Span, Spanned, Stmt as ChlStmt, VariantPayload as ChlVariantPayload, }, - interpreter::{DataSink, DataSourceDomainExtentImpl, http_server::SharedHttpServer}, + interpreter::{ + DataSink, DataSourceDomainExtentImpl, + http_server::{SharedHttpServer, UnopenedRoute, UnopenedRouteSink}, + }, }; mod comprehension; @@ -237,6 +240,35 @@ impl LoweringResult { // Lowering context // --------------------------------------------------------------------------- +/// What a pass may do with an `http_serve` route the source/sink registry does not +/// already hold. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Endpoints { + /// Open it. The pass is compiling the version that will serve it, and serving + /// an address the previous version did not is what adding an endpoint means. + #[default] + Open, + /// Leave it unopened, and lower it to an + /// [`UnopenedRoute`](crate::interpreter::http_server::UnopenedRoute). + /// + /// For a pass that answers a question rather than installing a version — a + /// `/diff`, or the planned tree the state guard reads. Its context is thrown + /// away, but a socket and a routing-table entry are not: opening one would + /// make *asking* change what the program serves, and would leave the port + /// taken when the pass that installs the version comes to bind it. + Inherited, +} + +/// One `http_serve` route as lowering knows it: its reply sink and the address +/// it serves. +#[derive(Clone)] +pub struct LoweredRoute { + pub(super) sink: Arc, + pub(super) port: u16, + pub(super) method: String, + pub(super) path: String, +} + /// Context for CHL → CCL lowering that carries registered data sources and sinks. /// /// Zero-argument function calls whose name appears in `sources` are lowered to @@ -277,6 +309,33 @@ pub struct LoweringContext { /// and reuse the per-port server. pub(super) shared_servers: HashMap>, + /// Every `http_serve` route this context knows: those seeded from the + /// registry and those this pass opened, by the route's source name. + /// + /// Keyed by *route* rather than by binding name, unlike + /// [`sink_bindings`](Self::sink_bindings): the response binding is a name the + /// program chooses and a new version may spell differently, whereas the route + /// is the address's identity. Carries the address as well as the sink, + /// because `sink()` is inherent on `HttpServerDataSource` and cannot be read + /// back off the erased `dyn DataSourceDomainExtentImpl` in + /// [`sources`](Self::sources), and because retiring a route needs the method + /// and path to unregister. + pub(super) http_routes: HashMap, + + /// Every `http_serve` route lowered *in this pass*, by source name. + /// + /// Separate from [`sources`](Self::sources) because that map is seeded with + /// the endpoints a previous version of the program opened + /// ([`SourceSinkRegistry`](crate::ccl::context::SourceSinkRegistry)), so a name + /// being present there means "already open", not "already lowered". Two + /// `http_serve` calls on one route within a single program remain an error; + /// re-lowering the same route in a later version is the reuse path. + pub(super) http_routes_this_pass: HashSet, + + /// What this pass may do with an `http_serve` naming a route the registry does + /// not already hold. + pub(super) endpoints: Endpoints, + /// Monotonic counter for minting unique synthetic names during lowering. /// Globally unique across nested scopes so inner binders cannot capture /// a reference inserted by an outer substitution. @@ -380,6 +439,60 @@ impl LoweringContext { out } + /// Every source registered in this context, for folding into the + /// [`SourceSinkRegistry`](crate::ccl::context::SourceSinkRegistry) that outlives + /// the compilation. Unlike [`take_sources`](Self::take_sources) this leaves + /// the map in place, so it may be called before the drain. + pub fn registered_sources( + &self, + ) -> impl Iterator>)> { + self.sources.iter().map(|(n, s)| (n.as_str(), s)) + } + + /// Every `http_serve` route this context knows, by source name. + pub fn registered_routes(&self) -> impl Iterator { + self.http_routes.iter().map(|(n, r)| (n.as_str(), r)) + } + + /// The routes this pass bound — every address the version being lowered + /// serves. A route the registry holds and this set omits is one the version + /// stopped serving. + pub fn routes_bound_this_pass(&self) -> &HashSet { + &self.http_routes_this_pass + } + + /// Hand over every bound TCP port's listener, leaving this context with none. + /// + /// Drained rather than cloned so that the registry it is folded into becomes + /// the sole owner. A copy left here would keep a retired port's listener alive + /// until the next pass replaced this context, which is a compilation later + /// than the pass that stopped serving it. Lowering is finished by the time + /// this is called, so nothing looks a port up afterwards. + pub fn take_servers(&mut self) -> impl Iterator)> + use<> { + std::mem::take(&mut self.shared_servers).into_iter() + } + + /// Answer against the endpoints this context already holds, opening none. + pub fn inherit_endpoints_only(&mut self) { + self.endpoints = Endpoints::Inherited; + } + + /// Seed this context with the sources and sinks a program already holds. + /// + /// A call naming one of these binds it; a call naming anything else opens + /// it. Both are allowed in every version — a replacement inherits what the + /// running program has and may add to it. + pub fn adopt_sources_and_sinks( + &mut self, + sources: impl IntoIterator>)>, + routes: impl IntoIterator, + servers: impl IntoIterator)>, + ) { + self.sources.extend(sources); + self.http_routes.extend(routes); + self.shared_servers.extend(servers); + } + /// Drain all sources accumulated for this compilation. /// /// Returns every source that was either pre-registered (e.g. stdin, test diff --git a/src/ccl/lower/stmts.rs b/src/ccl/lower/stmts.rs index 3e6931e70..64bae47b9 100644 --- a/src/ccl/lower/stmts.rs +++ b/src/ccl/lower/stmts.rs @@ -476,20 +476,8 @@ pub(super) fn lower_middle_stmt( format!("http_serve port must be a u16, got {port:?}"), ) })?; - // Share one tiny_http::Server per port across all http_serve routes. - if let std::collections::hash_map::Entry::Vacant(e) = ctx.shared_servers.entry(port_u16) - { - let server = SharedHttpServer::new(port_u16).map_err(|e| { - LoweringError::unsupported( - value.span, - format!("http_serve: failed to bind port {port_u16}: {e}"), - ) - })?; - e.insert(Arc::new(server)); - } - let server = ctx.shared_servers[&port_u16].clone(); let source_name = http_requests_source_name(&port, &method, &path); - if ctx.sources.contains_key(&source_name) { + if !ctx.http_routes_this_pass.insert(source_name.clone()) { return Err(LoweringError::unsupported( value.span, format!( @@ -497,14 +485,55 @@ pub(super) fn lower_middle_stmt( ), )); } - let source_obj = Rc::new(RefCell::new(HttpServerDataSource::new( - &server, - method.clone(), - path.clone(), - source_name.clone(), - ))); - let sink: Arc = source_obj.borrow().sink(); - ctx.sources.insert(source_name.clone(), source_obj); + // Bind an already-open route, or open a new one. A route the + // source/sink registry already holds is *inherited*: reusing its + // `HttpServerDataSource` keeps the listener, the routing-table entry + // and the requests buffered behind it, which is what lets a + // replacement version of the program pick up where this one left + // off. A route it does not hold is opened, whether this is the + // program's first version or a replacement — a version that adds an + // endpoint serves it as soon as the swap completes. + let sink: Arc = match ctx.http_routes.get(&source_name) { + Some(existing) => existing.sink.clone(), + None if ctx.endpoints == Endpoints::Inherited => { + let source_obj = Rc::new(RefCell::new(UnopenedRoute::new(source_name.clone()))); + ctx.sources.insert(source_name.clone(), source_obj); + Arc::new(UnopenedRouteSink) + } + None => { + // Share one tiny_http::Server per port across all http_serve routes. + if let std::collections::hash_map::Entry::Vacant(e) = + ctx.shared_servers.entry(port_u16) + { + let server = SharedHttpServer::new(port_u16).map_err(|e| { + LoweringError::unsupported( + value.span, + format!("http_serve: failed to bind port {port_u16}: {e}"), + ) + })?; + e.insert(Arc::new(server)); + } + let server = ctx.shared_servers[&port_u16].clone(); + let source_obj = Rc::new(RefCell::new(HttpServerDataSource::new( + &server, + method.clone(), + path.clone(), + source_name.clone(), + ))); + let sink: Arc = source_obj.borrow().sink(); + ctx.sources.insert(source_name.clone(), source_obj); + ctx.http_routes.insert( + source_name.clone(), + LoweredRoute { + sink: sink.clone(), + port: port_u16, + method: method.clone(), + path: path.clone(), + }, + ); + sink + } + }; let requests_expr = ctx.tag_machinery( Expr::new(TypedExprNode::Source(source_name.clone())), stmt.span, diff --git a/src/ccl/mut_elim.rs b/src/ccl/mut_elim.rs index 476ee51ce..4bfb1e71f 100644 --- a/src/ccl/mut_elim.rs +++ b/src/ccl/mut_elim.rs @@ -572,8 +572,7 @@ struct FeedSite { } /// `p ▷ .i : elt_ty` — projection of a tuple-typed variable (a writer-body -/// snapshot slot or the packed previous-values tuple). Mirrors -/// `transact_phase`'s `proj_tuple`. +/// snapshot slot). Mirrors `transact_phase`'s `proj_tuple`. fn proj_of(p: &Name, tuple_ty: &Type, i: usize, elt_ty: &Type) -> Expr { let mut proj = Expr::proj_index(i); proj.ty = Type::fun(tuple_ty.clone(), elt_ty.clone()); @@ -582,24 +581,25 @@ fn proj_of(p: &Name, tuple_ty: &Type, i: usize, elt_ty: &Type) -> Expr { app } -/// `__hist ≫ variant_project(`commit) ≫ .writes ≫ .i : domain ⇒ vty` — the -/// accumulator-`i` slice of the history's committing-write stream, built as one +/// `__hist ≫ variant_project(`commit) ≫ .writes ≫ .acc : domain ⇒ vty` — one +/// accumulator's slice of the history's committing-write stream, built as one /// flat compose so recognition (and the causal-slot grammar) match it -/// structurally. The ``variant_project(`commit)`` step eliminates the ``{`commit{𝑃} | `abort}`` decision to its dense payload before the `.writes` read. -fn writes_index_view( +/// structurally. The write set is keyed by accumulator name, so the slice is +/// named too. The ``variant_project(`commit)`` step eliminates the ``{`commit{𝑃} | `abort}`` decision to its dense payload before the `.writes` read. +fn writes_key_view( h: &Name, hist_ty: &Type, domain_ty: &Type, writes_ty: &Type, decision_ty: &Type, - i: usize, + acc: &str, vty: &Type, ) -> Expr { let payload_ty = crate::ccl::ccl_utils::commit_payload_ty(decision_ty); let vp = crate::ccl::ccl_utils::commit_project(decision_ty); let mut wproj = Expr::proj_field(F_WRITES); wproj.ty = Type::fun(payload_ty, writes_ty.clone()); - let mut iproj = Expr::proj_index(i); + let mut iproj = Expr::proj_field(acc); iproj.ty = Type::fun(writes_ty.clone(), vty.clone()); let mut comp = Expr::compose(vec![tvar(h, hist_ty.clone()), vp, wproj, iproj]); comp.ty = Type::fun(domain_ty.clone(), vty.clone()); @@ -725,8 +725,8 @@ pub(crate) fn hoist_feeds(mut body: Expr, feeds: Vec<(Name, Expr)>) -> Expr { /// induction and transaction bindings share ONE post-`lambda_elim` normal /// form (`(guard, source) ▷ zip ≫ body`), so recognition splits snapshot /// from body structurally and never rebuilds either. `writes` is always a -/// positional tuple (one element even for a single accumulator), matching -/// the transaction decision convention; the guard reads the *writes +/// record keyed by the variable written, matching the transaction decision +/// convention; the guard reads the *writes /// projection* of the history (causal — see `check_letrec_causal`); each /// feed rides the decision as a `to_` field, hoisted to /// `Feed(defer, __hist ≫ .to_)` for `channelize` to route. @@ -833,19 +833,19 @@ pub(crate) struct InductionFold { } impl InductionFold { - /// `__hist ≫ .writes ≫ .i : domain ⇒ vty` — accumulator `i`'s value stream. + /// `__hist ≫ .writes ≫ .acc : domain ⇒ vty` — accumulator `i`'s value stream. /// A per-position cross-domain read of that accumulator (`acc(pos)`) is this /// applied at `pos`; the transaction phase uses it to resolve a `commits(r)` /// decision that reads an induction accumulator at its request position. pub(crate) fn acc_view(&self, i: usize) -> Expr { - let (_, vty) = &self.accs[i]; - writes_index_view( + let (acc, vty) = &self.accs[i]; + writes_key_view( &self.hist, &self.hist_ty, &self.domain_ty, &self.writes_ty, &self.decision_ty, - i, + &acc.field_key(), vty, ) } @@ -872,9 +872,19 @@ pub(crate) fn fold_induction_loop( let (domain_ty, item_ty) = fun_parts(&iter.ty); let acc_tys: Vec = accs.iter().map(|(_, t)| t.clone()).collect(); - // The proposed write set — always a positional tuple, one element even - // for a single accumulator (the transaction decision convention). - let writes_ty = Type::Tuple(acc_tys.clone()); + // The proposed write set, labelled by the accumulators' own names. A + // positional tuple reads the same everywhere downstream and costs nothing + // here, but the labels are the only record of which variable a slot belongs + // to: `planning::plan_loops` is handed this and nothing else, so an + // unlabelled write set leaves it inventing `acc0`/`acc1` from position, and + // a slot identified only by position cannot be matched back to its variable + // by anything downstream — including a second compilation of the same + // program, which is what makes the labels worth carrying. + let writes_ty = Type::Record( + accs.iter() + .map(|(n, t)| (n.field_key(), t.clone())) + .collect(), + ); let h = Name::fresh("__hist"); let r = Name::fresh("__pos"); @@ -944,7 +954,11 @@ pub(crate) fn fold_induction_loop( // history is a causal reference (see `check_letrec_causal`); the // defaults are the accumulators' pre-loop bindings, tupled. let writes_view = hist_field_view(&h, &hist_ty, &domain_ty, F_WRITES, &writes_ty, &decision_ty); - let mut defaults = Expr::tuple(accs.iter().map(|(n, ty)| tvar(n, ty.clone())).collect()); + let mut defaults = Expr::new(TypedExprNode::Record( + accs.iter() + .map(|(n, ty)| (n.field_key(), tvar(n, ty.clone()))) + .collect(), + )); defaults.ty = writes_ty.clone(); let guard = { let mut arg = Expr::tuple(vec![writes_view, tvar(&r, domain_ty.clone()), defaults]); @@ -960,9 +974,20 @@ pub(crate) fn fold_induction_loop( app }; - // λ r → let __prev = ⟨guard⟩ in (__prev.0, …, r ▷ iter) ▷ __body - let mut snap_elts: Vec = (0..accs.len()) - .map(|i| proj_of(&prev, &writes_ty, i, &acc_tys[i])) + // λ r → let __prev = ⟨guard⟩ in (__prev.acc₀, …, r ▷ iter) ▷ __body. + // The previous values are the write set, so they are read by accumulator + // name; the body's parameter stays a positional tuple, which is what + // `transact_phase::build_writer` and the drive both build. + let mut snap_elts: Vec = accs + .iter() + .zip(&acc_tys) + .map(|((acc, _), vty)| { + let mut proj = Expr::proj_field(acc.field_key()); + proj.ty = Type::fun(writes_ty.clone(), vty.clone()); + let mut app = Expr::apply(tvar(&prev, writes_ty.clone()), proj); + app.ty = vty.clone(); + app + }) .collect(); let mut item_read = Expr::apply(tvar(&r, domain_ty.clone()), iter.clone()); item_read.ty = item_ty.clone(); @@ -982,8 +1007,16 @@ pub(crate) fn fold_induction_loop( // The read's default is the accumulator's pre-loop binding. let mut reads: Vec<(TypedBinding, Expr)> = Vec::new(); let mut renames: Vec<(Name, Name)> = Vec::new(); - for (i, (acc, vty)) in accs.iter().enumerate() { - let view = writes_index_view(&h, &hist_ty, &domain_ty, &writes_ty, &decision_ty, i, vty); + for (acc, vty) in accs.iter() { + let view = writes_key_view( + &h, + &hist_ty, + &domain_ty, + &writes_ty, + &decision_ty, + &acc.field_key(), + vty, + ); let view_ty = view.ty.clone(); let mut arg = Expr::tuple(vec![view, tvar(acc, vty.clone())]); arg.ty = Type::Tuple(vec![view_ty, vty.clone()]); @@ -1492,12 +1525,12 @@ fn decision_writes(dec: &Expr) -> Vec { .find(|(f, _)| f == F_WRITES) .expect("letrec phase: a writer decision has a `writes` field") .1; - let TypedExprNode::Tuple(elts) = &writes.node else { - panic!("letrec phase: a decision `writes` is a positional tuple"); + let TypedExprNode::Record(elts) = &writes.node else { + panic!("letrec phase: a decision `writes` is keyed by accumulator"); }; return elts .iter() - .map(|e| Subst::discharge_env_in_place(e.clone(), &env)) + .map(|(_, e)| Subst::discharge_env_in_place(e.clone(), &env)) .collect(); } _ => panic!( @@ -1584,9 +1617,26 @@ fn conditional_decision( decision_record(commit, write_elts, writes_ty) } -/// Assemble a writer decision record `{commit, writes: (write_elts…)}`. +/// Assemble a writer decision record `{commit, writes: {acc: e, …}}`. +/// +/// The write set's labels come from `writes_ty`, which is where the +/// accumulators' names live once [`fold_induction_loop`] has built it. fn decision_record(commit: Expr, write_elts: Vec, writes_ty: &Type) -> Expr { - let mut writes = Expr::tuple(write_elts); + let Type::Record(fields) = writes_ty else { + panic!("decision write set is a record keyed by accumulator name, got {writes_ty}"); + }; + assert_eq!( + fields.len(), + write_elts.len(), + "one written value per accumulator" + ); + let mut writes = Expr::new(TypedExprNode::Record( + fields + .iter() + .map(|(label, _)| label.clone()) + .zip(write_elts) + .collect(), + )); writes.ty = writes_ty.clone(); let mut rec = Expr::new(TypedExprNode::Record(vec![ (COMMIT_SELECTOR.to_string(), commit), diff --git a/src/ccl/names.rs b/src/ccl/names.rs index 0797e610a..f263f2aed 100644 --- a/src/ccl/names.rs +++ b/src/ccl/names.rs @@ -341,13 +341,13 @@ impl Name { /// and below loop planning unusable — see `src/ccl/design/diffing.md`. /// /// The per-record uniqueness this relies on is a property of spellings, not - /// of construction: a key spelling is either the user's own variable name, - /// distinct within its block, or a label planning mints indexed by position - /// (`acc0`, `acc1`), and a writer's `to__` taps share the record - /// with both. Nothing in the type system rules a collision out, so each site - /// that builds a record from these labels asserts distinctness in debug: - /// `hist_record` in `planning/loops.rs`, and the two `keys_map` inserts - /// in `interpreter/operator_conversion.rs`. + /// of construction: a key spelling is the user's own variable name, distinct + /// within the block or loop that declares it, or a writer's reply tap + /// (`to__`), which shares the record with them. Nothing in the type + /// system rules a collision out, so each site that builds a record from these + /// labels asserts distinctness in debug: `hist_record` in + /// `planning/loops.rs`, and the two `keys_map` inserts in + /// `interpreter/operator_conversion.rs`. pub fn field_key(&self) -> String { match self { // Not a binder, so no mutable variable is ever declared at one and diff --git a/src/ccl/planning/loops.rs b/src/ccl/planning/loops.rs index 26d50b39c..873504009 100644 --- a/src/ccl/planning/loops.rs +++ b/src/ccl/planning/loops.rs @@ -603,8 +603,8 @@ fn recognize_group(h: TypedBinding, def: Expr, letrec_body: Expr) -> Expr { matches!(which, Builtin::GetPrevSeq), "letrec recognition: induction history causal by get_prev_txn" ); - let TypedExprNode::Tuple(inits) = defaults.node else { - panic!("letrec recognition: guard defaults are not the tupled inits"); + let TypedExprNode::Record(inits) = defaults.node else { + panic!("letrec recognition: guard defaults are not the accumulators' inits record"); }; let (prev_slots, source, writer_body) = split_decision_compose(*applied, &decision_ty); @@ -621,18 +621,16 @@ fn recognize_group(h: TypedBinding, def: Expr, letrec_body: Expr) -> Expr { }) .collect(); - // One mutable variable key per accumulator. Every read is positional - // (`__hist ≫ .writes ≫ .i`), so these names carry no meaning beyond - // labelling the mutable variable record — but the label still has to be - // distinct *within* that record, and `field_key` is the plain spelling. So - // index by position: a shared `"acc"` base would collapse two accumulators - // onto one field, and position is the one distinguisher that is also stable - // across compilations, which uid-free labels require. + // One store key per accumulator, under the name the program gave it. + // `mut_elim` labels the write set by `field_key`, so the accumulators arrive + // named and stay named: a read is `__hist ≫ .writes ≫ .acc`, the history + // record is keyed the same way, and two compilations of one program agree on + // which slot is which variable — which is what lets a replacement version + // resume an accumulator rather than guess by position. let keys: Vec = inits .into_iter() - .enumerate() - .map(|(i, init)| TransactKey { - name: Name::fresh(format!("acc{i}")), + .map(|(label, init)| TransactKey { + name: Name::fresh(label), init, }) .collect(); @@ -700,7 +698,7 @@ fn hist_field_read(hist: &Name, hist_ty: &Type, field: String, field_ty: Type) - /// Rewrite every `__hist` view in the letrec body to a history-record /// projection `__hist.field`. The phase builds accumulator reads as the flat -/// compose `__hist ≫ .writes ≫ .i` and feed reads as `__hist ≫ .to_`; +/// compose `__hist ≫ .writes ≫ .acc` and feed reads as `__hist ≫ .to_`; /// downstream normalization may extend those composes (`__hist ≫ .to ≫ f`), /// so the match is on the *prefix*, keeping any tail elements. fn rewrite_hist_reads( @@ -727,13 +725,22 @@ fn rewrite_hist_reads( // elements the prefix covered (the `variant_project` step included). let replacement: Option<(Expr, usize)> = match (elts.get(2).map(|x| &x.node), elts.get(3).map(|x| &x.node)) { + // An accumulator read `` __hist ≫ variant_project(`commit) ≫ + // .writes ≫ .acc ``. Both projections are named now that the + // write set is keyed by accumulator, so `.writes` on the outer + // one is what tells this from a tap read. ( Some(TypedExprNode::Proj(ProjKey::Field(f))), - Some(TypedExprNode::Proj(ProjKey::Index(i))), + Some(TypedExprNode::Proj(ProjKey::Field(acc))), ) if f == F_WRITES => { - let field = keys[*i].name.field_key(); - let field_ty = Type::fun(domain_ty.clone(), acc_tys[*i].clone()); - Some((hist_field_read(hist, hist_ty, field, field_ty), 4)) + let i = keys + .iter() + .position(|k| k.name.field_key() == *acc) + .unwrap_or_else(|| { + panic!("letrec recognition: `.writes ≫ .{acc}` names no accumulator") + }); + let field_ty = Type::fun(domain_ty.clone(), acc_tys[i].clone()); + Some((hist_field_read(hist, hist_ty, acc.clone(), field_ty), 4)) } (Some(TypedExprNode::Proj(ProjKey::Field(f))), _) if f != F_WRITES => { // A tap read ``__hist ≫ variant_project(`commit) ≫ .to_``: diff --git a/src/ccl/transact_phase.rs b/src/ccl/transact_phase.rs index 6f00c851f..794ef352c 100644 --- a/src/ccl/transact_phase.rs +++ b/src/ccl/transact_phase.rs @@ -2090,21 +2090,28 @@ fn build_writer( ); let commit = crate::ccl::ccl_utils::disjoin(commit_paths, true, &Type::Base(BaseType::Bool)); - // The decision `writes` is a positional tuple over `write_keys`, matching - // `emit_transact_writer` (a single write is a one-element tuple). A write key - // never assigned in the block keeps its snapshot (unchanged). + // The decision `writes` is keyed by the variables written, matching + // `emit_transact_writer`. A write key never assigned in the block keeps its + // snapshot (unchanged). let write_tys: Vec = site.write_keys.iter().map(value_ty).collect(); - let write_vals: Vec = site + let write_fields: Vec<(String, Expr)> = site .write_keys .iter() .map(|wk| { - env.get(wk).cloned().unwrap_or_else(|| { + let val = env.get(wk).cloned().unwrap_or_else(|| { panic!("transact_phase: write key `{wk}` never assigned in its block") - }) + }); + (wk.field_key(), val) }) .collect(); - let mut writes = Expr::tuple(write_vals); - writes.ty = Type::Tuple(write_tys.clone()); + let mut writes = Expr::new(TypedExprNode::Record(write_fields)); + writes.ty = Type::Record( + site.write_keys + .iter() + .map(|wk| wk.field_key()) + .zip(write_tys.clone()) + .collect(), + ); // Decision record `{commit, writes, to_*}` — built by the shared // `writer_decision_record` (the one place the tap/`__fire` encoding lives, so @@ -2165,7 +2172,7 @@ fn proj_item(item: &Expr, item_ty: &Type, i: usize, elt_ty: &Type) -> Expr { } /// The writer source extended to carry each cross-read accumulator at its request -/// position: `λ x → (source(x), acc0-view(x), …) : dom ⇒ (item, v0, …)`. +/// position: `λ x → (source(x), acc-view(x), …) : dom ⇒ (item, v0, …)`. /// `lambda_elim` point-frees it to a `zip`; recognition lifts it verbatim as the /// writer source, and op-conversion (`build_commit_store`) destructures the `zip` /// to co-iterate the accumulator streams alongside the loop source. @@ -2589,7 +2596,7 @@ fn per_key_view( dom: &Type, rec_ty: &Type, decision_ty: &Type, - idx: usize, + key: &str, value_ty: &Type, view_rec_ty: &Type, ) -> Expr { @@ -2609,8 +2616,8 @@ fn per_key_view( ]); time_view.ty = Type::fun(dom.clone(), Type::Txn); - // write leg: commits_j ≫ .decision ≫ variant_project(`commit) ≫ .writes ≫ .idx. - let mut iproj = Expr::proj_index(idx); + // write leg: commits_j ≫ .decision ≫ variant_project(`commit) ≫ .writes ≫ .key. + let mut iproj = Expr::proj_field(key); iproj.ty = Type::fun(writes_ty.clone(), value_ty.clone()); let mut write_view = Expr::compose(vec![ tvar(commits_j, commits_ty), @@ -2899,13 +2906,13 @@ fn plan_store( Some(sites) => { let taps: Vec = sites .iter() - .map(|&(j, idx)| { + .map(|&(j, _idx)| { per_key_view( &commits[j], &site_dom[j], &commit_rec_ty[j], &site_decision_ty[j], - idx, + &k.field_key(), &v, &view_rec_ty, ) diff --git a/src/control_port.rs b/src/control_port.rs new file mode 100644 index 000000000..03fdc6aef --- /dev/null +++ b/src/control_port.rs @@ -0,0 +1,398 @@ +//! Control port: HTTP endpoints for diffing a running program against a new +//! version of its source, and for replacing it with that version. +//! +//! Two endpoints, both taking the new source as their argument: +//! +//! - `/diff` — how the new version differs from the running one, rendered as an +//! annotated tree. Answers the question without changing anything. +//! - `/update` — replace the running program with the new version. +//! +//! The source may be the whole query string, percent-decoded, or a `POST` body. +//! A `phase=` parameter selects where in the pipeline `/diff` compares (see +//! [`OFFERED_PHASES`]); it must come first when the query also carries the +//! source. +//! +//! # Why requests are handed to the main loop +//! +//! Nothing on the interpreter side is [`Send`]: the operator graph, the sources, +//! and the compilation contexts are `Rc`/`RefCell` throughout, and compiling a +//! new version needs all three. So the server thread does no compilation. It +//! parses the request into a [`ControlRequest`], sends it to the main loop over +//! a channel, and blocks on the reply — the main loop services it between ticks, +//! where it already holds the program exclusively. + +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +use std::thread; + +use log::info; + +use crate::ccl::context::Phase; + +/// What a control-port client asked for. +pub enum ControlRequest { + /// Report how `code` differs from the running program, comparing at `phase`. + Diff { code: String, phase: Phase }, + /// Replace the running program with `code`. + Update { code: String }, +} + +/// The answer to one [`ControlRequest`], as an HTTP status and a plain-text body. +#[derive(Debug)] +pub struct ControlReply { + pub status: u16, + pub body: String, +} + +impl ControlReply { + /// A `200` carrying `body`. + pub fn ok(body: impl Into) -> Self { + Self { + status: 200, + body: body.into(), + } + } + + /// A `400` carrying `body` — the request named a version the running + /// program cannot compile or cannot adopt. + pub fn rejected(body: impl Into) -> Self { + Self { + status: 400, + body: body.into(), + } + } +} + +/// One request awaiting an answer, together with the channel the server thread +/// is blocked on. +/// +/// Consumed by [`answer`](Self::answer), so a serviced request cannot be left +/// unanswered by accident; dropping one instead unblocks the server thread with +/// a `503`. +pub struct ControlMessage { + request: ControlRequest, + reply: Option>, +} + +impl ControlMessage { + /// What was asked. + pub fn request(&self) -> &ControlRequest { + &self.request + } + + /// Answer the request and release the server thread. + pub fn answer(mut self, reply: ControlReply) { + if let Some(tx) = self.reply.take() { + let _ = tx.send(reply); + } + } +} + +impl Drop for ControlMessage { + fn drop(&mut self) { + if let Some(tx) = self.reply.take() { + let _ = tx.send(ControlReply { + status: 503, + body: "control request dropped without an answer\n".to_string(), + }); + } + } +} + +/// The main loop's end of the control port. +/// +/// Holding one keeps the server thread's channel open; dropping it makes every +/// later request fail rather than hang. +pub struct ControlPort { + rx: Receiver, +} + +impl ControlPort { + /// Start the control server on `port`. + pub fn new(port: u16) -> Self { + let (tx, rx) = sync_channel::(0); + + thread::spawn(move || { + let server = tiny_http::Server::http(format!("0.0.0.0:{port}")) + .expect("Failed to start control port server"); + info!("Control port running at http://localhost:{port}"); + + for mut request in server.incoming_requests() { + let mut body = String::new(); + let _ = std::io::Read::read_to_string(request.as_reader(), &mut body); + let reply = match parse_request(request.url(), &body) { + Err(reply) => reply, + Ok(parsed) => { + let (reply_tx, reply_rx) = sync_channel::(0); + let message = ControlMessage { + request: parsed, + reply: Some(reply_tx), + }; + // A closed channel means the program is gone; a closed + // reply channel means the main loop dropped the message + // without the `Drop` answer arriving, which is a bug + // rather than a state to report differently. + match tx.send(message) { + Err(_) => ControlReply { + status: 503, + body: "program is not accepting control requests\n".to_string(), + }, + Ok(()) => reply_rx.recv().unwrap_or(ControlReply { + status: 500, + body: "control request was never answered\n".to_string(), + }), + } + } + }; + let header: tiny_http::Header = + "Content-Type: text/plain; charset=utf-8".parse().unwrap(); + let _ = request.respond( + tiny_http::Response::from_string(reply.body) + .with_status_code(reply.status) + .with_header(header), + ); + } + }); + + ControlPort { rx } + } + + /// Take the next pending request, or `None` if none is waiting. + /// + /// Non-blocking: the main loop calls this at a tick boundary and carries on + /// when nothing is queued. + pub fn poll(&self) -> Option { + self.rx.try_recv().ok() + } +} + +/// Every pipeline position `/diff` offers, with the `phase=` spelling that names +/// it. +/// +/// The compiler can stop at any [`Phase`]'s output; this table is the subset the +/// control port offers, and it leaves out the three that answer no question a +/// caller of `/diff` has — `uniquify`, whose tree diffs identically to `lowered` +/// because the hash is uid-robust, and `transact`/`letrec`, which are the two +/// halves of one rewrite and report a shape mid-rewrite. +/// +/// One table rather than a lookup beside a list of spellings, so the set a +/// caller may name, the set the rejection diagnostic offers, and the set +/// `every_offered_phase_is_a_diff_point` exercises are the same set. +pub const OFFERED_PHASES: &[(&str, Phase)] = &[ + ("lowered", Phase::Lower), + ("inferred", Phase::Infer), + ("inlined", Phase::Inline), + ("channelized", Phase::Channelize), + ("as-of-read", Phase::AsOfRead), + ("lambda-elim", Phase::LambdaElim), + ("planned", Phase::Planning), +]; + +/// The phase `/diff` compares at when the request names none: `as-of-read`, the +/// last position at which the tree still has binders, and the one `lambda_elim` +/// consumes. The mutability and channelization rewrites are complete there and +/// an edit still localizes the way it does further up — see +/// `src/ccl/design/diffing.md`, "Which phase to diff". +pub const DEFAULT_PHASE: Phase = Phase::AsOfRead; + +/// The phase a `phase=` spelling names, or `None` when it names none. +pub fn phase_from_name(name: &str) -> Option { + OFFERED_PHASES + .iter() + .find(|(spelling, _)| *spelling == name) + .map(|(_, phase)| *phase) +} + +/// Every `phase=` spelling, for a diagnostic. +fn phase_names() -> String { + OFFERED_PHASES + .iter() + .map(|(spelling, _)| *spelling) + .collect::>() + .join(", ") +} + +/// Split a URL into its path and its raw query string. +fn split_url(url: &str) -> (&str, &str) { + match url.split_once('?') { + Some((path, query)) => (path, query), + None => (url, ""), + } +} + +/// Parse a request into the [`ControlRequest`] the main loop services, or the +/// reply to send when it is not one. +fn parse_request(url: &str, body: &str) -> Result { + let (path, query) = split_url(url); + let (phase_name, rest) = split_phase_param(query); + + // The source is the body when there is one, so a program containing `&` or + // `#` need not be percent-encoded to survive the query string. + let code = if body.trim().is_empty() { + percent_decode(rest) + } else { + body.to_string() + }; + + match path { + "/diff" => { + let phase = match phase_name { + None => DEFAULT_PHASE, + Some(name) => phase_from_name(name).ok_or_else(|| { + ControlReply::rejected(format!( + "unknown phase {name:?}; expected one of: {}\n", + phase_names() + )) + })?, + }; + require_code(&code)?; + Ok(ControlRequest::Diff { code, phase }) + } + "/update" => { + require_code(&code)?; + Ok(ControlRequest::Update { code }) + } + _ => Err(ControlReply { + status: 404, + body: "endpoints: /diff?, /update?\n".to_string(), + }), + } +} + +fn require_code(code: &str) -> Result<(), ControlReply> { + if code.trim().is_empty() { + return Err(ControlReply::rejected( + "no source given: pass it as the query string or the request body\n", + )); + } + Ok(()) +} + +/// Peel a leading `phase=&` off a query string. +/// +/// Leading rather than anywhere, because everything after it is the source +/// program and an `&` inside a program must not be read as a parameter +/// separator. +fn split_phase_param(query: &str) -> (Option<&str>, &str) { + let Some(rest) = query.strip_prefix("phase=") else { + return (None, query); + }; + match rest.split_once('&') { + Some((name, code)) => (Some(name), code), + None => (Some(rest), ""), + } +} + +/// Decode `application/x-www-form-urlencoded` text: `%XX` escapes and `+` for +/// space. +/// +/// An incomplete or non-hex `%` escape is left as written rather than rejected — +/// a bare `%` is a modulus in CHL, so a client that percent-encoded nothing +/// still gets its program through. +fn percent_decode(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'+' => { + out.push(b' '); + i += 1; + } + b'%' if i + 2 < bytes.len() => { + let hex = std::str::from_utf8(&bytes[i + 1..i + 3]).ok(); + match hex.and_then(|h| u8::from_str_radix(h, 16).ok()) { + Some(byte) => { + out.push(byte); + i += 3; + } + None => { + out.push(bytes[i]); + i += 1; + } + } + } + b => { + out.push(b); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn diff_of(url: &str) -> (String, Phase) { + match parse_request(url, "").expect("parses") { + ControlRequest::Diff { code, phase } => (code, phase), + ControlRequest::Update { .. } => panic!("expected a diff request"), + } + } + + #[test] + fn diff_defaults_to_the_phase_before_lambda_elimination() { + let (code, phase) = diff_of("/diff?x%20%3D%201%3B%20x"); + assert_eq!(code, "x = 1; x"); + assert_eq!(phase, Phase::AsOfRead); + } + + #[test] + fn a_leading_phase_parameter_selects_the_diff_point() { + let (code, phase) = diff_of("/diff?phase=inferred&x = 1; x"); + assert_eq!(code, "x = 1; x"); + assert_eq!(phase, Phase::Infer); + } + + /// A `&` after the source's first character is part of the program, not a + /// parameter separator — only a *leading* `phase=` is peeled. + #[test] + fn an_ampersand_in_the_source_is_not_a_parameter_separator() { + let (code, _) = diff_of("/diff?a = 1 & 2; a"); + assert_eq!(code, "a = 1 & 2; a"); + } + + #[test] + fn a_bare_percent_survives_decoding() { + let (code, _) = diff_of("/diff?x = 7 % 3; x"); + assert_eq!(code, "x = 7 % 3; x"); + } + + #[test] + fn a_body_supplies_the_source_when_the_query_does_not() { + let request = parse_request("/update", "y = 2; y").expect("parses"); + match request { + ControlRequest::Update { code } => assert_eq!(code, "y = 2; y"), + ControlRequest::Diff { .. } => panic!("expected an update request"), + } + } + + #[test] + fn an_unknown_phase_is_rejected_rather_than_defaulted() { + let reply = parse_request("/diff?phase=nonsense&x", "") + .err() + .expect("rejected"); + assert_eq!(reply.status, 400); + assert!(reply.body.contains("channelized"), "{}", reply.body); + } + + #[test] + fn a_request_with_no_source_is_rejected() { + let reply = parse_request("/diff", "").err().expect("rejected"); + assert_eq!(reply.status, 400); + } + + #[test] + fn an_unserviced_message_unblocks_its_caller() { + let (tx, rx) = sync_channel::(0); + let waiter = thread::spawn(move || rx.recv().expect("a reply").status); + drop(ControlMessage { + request: ControlRequest::Update { + code: "x".to_string(), + }, + reply: Some(tx), + }); + assert_eq!(waiter.join().unwrap(), 503); + } +} diff --git a/src/interpreter/commit_operator.rs b/src/interpreter/commit_operator.rs index abc8f7610..f27d52f2b 100644 --- a/src/interpreter/commit_operator.rs +++ b/src/interpreter/commit_operator.rs @@ -126,6 +126,29 @@ impl CommitEngine { } } + /// Create an engine seeded like [`new`](Self::new) whose clock starts at + /// `at + 1`, for a store whose first iteration position is `at`. + /// + /// A store replacing one in a running program starts where its predecessor + /// had reached. The drive maps position `p` to tick `p + 1` and reads the + /// previous accumulator as of tick `p`, so starting the clock at `at + 1` + /// makes `at` this store's first position while leaving the + /// position-to-tick correspondence the dense read shares with the drive + /// intact. + /// + /// The seed still sits at tick `0`, so a position the predecessor decided + /// folds to the value it handed over rather than to nothing. This store has + /// no record of what that position actually held — the value is the one the + /// predecessor ended on — but a reader enumerating a fixed collection asks + /// about every position of it, and the last value is the one such a read is + /// after. + pub fn seeded_at(at: CommitTs, init: HashMap) -> Self { + Self { + next_ts: at + 1, + ..Self::new(init) + } + } + /// An empty engine for a **position-driven induction store**: no tick-0 init /// seed (the accumulator's init is the reader's fold default, supplied by /// `get_prev_seq`), driven by [`step`](Self::step) rather than @@ -1115,52 +1138,17 @@ impl TileProducer for CommitProducer { } } -/// Extract a source stream's codomain elements (the items to transact over) in -/// the codomain's **column order**. -/// -/// This is correct for the **commit writer** precisely because transactions are -/// *unordered* — each item becomes a commit proposal the [`CommitOperator`] -/// serializes by frontier/conflict, and any serialization is a valid commit order -/// (see the unordered-mutability design commitment). So an async source whose -/// domain arrives out of position order (a `HashMap` enumeration) may be processed -/// in arrival order without affecting the result. -/// -/// The **induction** driver must NOT use this: its recurrence `xₙ = f(xₙ₋₁, itemₙ)` -/// is position-ordered, so it reads by absolute domain position via -/// [`decode_source_positioned`], which sorts. The two look alike but carry opposite -/// ordering requirements — do not swap one for the other. -fn decode_source_items(tile: &Tile) -> Vec { - let Tile::SealedFunction { - domain, codomain, .. - } = tile - else { - return Vec::new(); - }; - match codomain.as_ref() { - Tile::Scalar(cv) => (0..domain.len()).map(|i| cv.index_at(i)).collect(), - // A cross-domain co-iterated source `zip((item, acc(r), …))`: each position - // is a `Record` of the scalar columns. The writer body reads the loop item - // off `._0` and each threaded induction accumulator off its own field — - // the shape `build_writer` lays out for a commit decision that reads an - // accumulator at its request position. - Tile::Record(_) => (0..domain.len()) - .map(|i| source_value_at(codomain, i)) - .collect(), - _ => Vec::new(), - } -} - /// Decode an iteration source tile into `(absolute domain position, item)` pairs, -/// **sorted by position** — the ordered counterpart of [`decode_source_items`]. +/// **sorted by position**. /// -/// The induction driver's recurrence is position-ordered, so it cannot use column -/// order: an **async** source's domain arrives *unordered* (it enumerates a set of -/// arrived keys) and *compacts* as its consumed prefix is released, so column order -/// is not position order. Pairing each item with its actual `UInt` domain position -/// and sorting makes the driver read `x₀, x₁, …` in order regardless of arrival. A -/// finite list is the special case (its domain is already `[0, 1, …]`). Contrast -/// [`decode_source_items`], which the *transaction* writer uses because commit -/// order is unordered. +/// Both drivers read their source through this. An **async** source's domain +/// arrives *unordered* (it enumerates a set of arrived keys) and *compacts* as its +/// consumed prefix is released, so a column index is neither a domain position nor +/// stable across a release. Pairing each item with its actual `UInt` domain +/// position gives both drivers a name for an item that outlives the view it was +/// read from — which the induction recurrence needs to run `x₀, x₁, …` in order, +/// and the transaction driver needs to say which items it has finished. A finite +/// list is the special case (its domain is already `[0, 1, …]`). fn decode_source_positioned(tile: &Tile) -> Vec<(usize, Value)> { let Tile::SealedFunction { domain, codomain, .. @@ -1229,6 +1217,10 @@ pub struct InductionStore { /// Keys written, in decision-`writes` order: the accumulator mutable variables, then /// any reply-tap (`to_`) keys. write_keys: Vec, + /// The tick this store's seed sits at, and so the first position it decides. + /// `0` for a store that starts with its source; the resume position for one + /// replacing a store in a running program. + resume_at: CommitTs, /// Reply-tap decision fields, appended to each write set (see /// [`body_decision_at`]). Empty for a store with no feed. tap_fields: Vec, @@ -1245,6 +1237,7 @@ impl InductionStore { tap_fields: Vec, key_extent: Extent, value_extent: Extent, + resume_at: CommitTs, ) -> Self { let output_tiling = full_store_tiling(&key_extent, &value_extent); Self { @@ -1253,6 +1246,7 @@ impl InductionStore { write_keys, tap_fields, output_tiling, + resume_at, } } @@ -1317,8 +1311,10 @@ impl TileOperator for InductionStore { // self-describing: `read_as_of`/`store_value_at` fold to the init below // the first *iteration* change (a leading carry) without an external // default. Iterations therefore occupy ticks 1.., a `+ 1` offset the - // driver and the dense read both apply. - engine: CommitEngine::new(inits), + // driver and the dense read both apply. A store that resumes starts + // its clock at the position it resumes at; a store that starts with + // its source resumes at `0`, which is the same seeding. + engine: CommitEngine::seeded_at(self.resume_at, inits), body_producer, write_keys: self.write_keys.clone(), tap_fields: self.tap_fields.clone(), @@ -1389,7 +1385,7 @@ impl TileProducer for InductionStoreProducer { let started_at = self.processed(); while let Some(pos) = next_decided_position(&body_tile, self.processed()) { let Some((commit, writes, tap_fired)) = - body_decision_at(&body_tile, pos, &self.tap_fields) + body_decision_at(&body_tile, pos, &self.write_keys, &self.tap_fields) else { break; }; @@ -2785,6 +2781,8 @@ pub struct InductionDriver { read_keys: Vec, read_extents: Vec, item_extent: Extent, + /// The first position this driver emits at. See [`DriverWindow::new`]. + resume_at: usize, } impl InductionDriver { @@ -2794,6 +2792,7 @@ impl InductionDriver { read_keys: Vec, read_extents: Vec, item_extent: Extent, + resume_at: usize, ) -> Self { debug_assert_eq!( read_keys.len(), @@ -2807,6 +2806,7 @@ impl InductionDriver { read_keys, read_extents, item_extent, + resume_at, } } } @@ -2836,8 +2836,20 @@ impl TileOperator for InductionDriver { wakeups: scheduler.wakeup_queue(), read_keys: self.read_keys.clone(), window: DriverWindow::new(self.read_extents.clone(), self.item_extent.clone()), - emitted_through: None, - source_released_through: None, + // A resuming driver has already emitted every position below the one + // its store resumes at — by its predecessor, whose rows are gone. The + // item cursor is where that is said: it is what the next position to + // iterate is taken from, and what the store's frontier is checked + // against. `None` for a driver starting at `0`, which has emitted + // nothing. + emitted_through: self.resume_at.checked_sub(1), + // And inherits the release cursor with it. A resuming driver has no + // interest in the prefix below the position it starts at, which is + // what this cursor records; leaving it empty would have this driver + // re-release a prefix its predecessor already released, and would + // read a position the source re-offers there as an out-of-order + // arrival. + source_released_through: self.resume_at.checked_sub(1), source_fully_released: false, }) } @@ -3149,8 +3161,14 @@ struct TransactDriverProducer { /// requests its own re-pull instead of looping inside `get`. wakeups: WakeupQueue, read_keys: Vec, - /// The source item being attempted. Advanced only by `release` — the - /// writer's ack that an attempt finished. + /// The **absolute source position** of the item being attempted. Every + /// position below it has finished, because the drive always attempts the + /// lowest position the source still offers. + /// + /// Absolute rather than a count of the columns the source currently offers: + /// a column count names a position in a view, so it means nothing to a drive + /// that did not emit it, and a replacement drive taking over a running + /// program would re-attempt every transaction the retired one committed. current: usize, /// The emitted rows — the attempts in flight, including superseded retries /// not yet reclaimed. @@ -3233,7 +3251,13 @@ impl TileProducer for TransactDriverProducer { // (a list) is terminal on the first pull; a live source (an HTTP request // stream) never is, so a momentarily drained one must not read as done. let source_complete = src.is_terminal(); - let items = decode_source_items(&src); + // Positioned, so an item is named by where it sits in the source's own + // domain rather than by where it sits in the columns still on offer. The + // lowest position at or above the cursor is the next item: the cursor is + // the attempt in flight until its ack, and the ack both advances it and + // withdraws the position from the source. + let items = decode_source_positioned(&src); + let next_item = items.iter().find(|(pos, _)| *pos >= self.current); let store = self .store_producer .get(self.store_producer.tiling().universal_guard()); @@ -3247,11 +3271,12 @@ impl TileProducer for TransactDriverProducer { .map(|k| store_current(&store, k).map(|(_, v)| v)) .collect(); - if self.current < items.len() + if let Some((pos, item)) = next_item && let Some(frontier) = frontier - && self.latest_emit != Some((self.current, frontier)) + && self.latest_emit != Some((*pos, frontier)) { - let item = items[self.current].clone(); + let (pos, item) = (*pos, item.clone()); + self.current = pos; // The body reads snapshot position `i` as `p.i`. A read key with no // value yet gets the item as a stand-in of the right extent. Load- // bearing assumption: a body that writes an *absent* key is @@ -3275,7 +3300,7 @@ impl TileProducer for TransactDriverProducer { // "all transactions attempted" is exactly this tile closing. A live // window that is momentarily empty over an incomplete source stays // non-terminal — the drained-but-live case. - let done = source_complete && self.current >= items.len(); + let done = source_complete && next_item.is_none(); // Re-arm while a transaction remains to attempt. It covers every // continuation uniformly: an attempt awaiting its commit-ack, a retry // waiting for the frontier to move, and the first pull of all — where the @@ -3283,7 +3308,7 @@ impl TileProducer for TransactDriverProducer { // an attempt against yet. A writer that is *drained but live* does not // re-arm: a future arrival wakes it through the source, so re-arming // would busy-poll an idle server. - if self.current < items.len() { + if next_item.is_some() { self.wakeups.request(self.consumer.clone()); } self.window.render(done) @@ -3310,7 +3335,19 @@ impl TileProducer for TransactDriverProducer { if let Some((pos, row)) = self.window.newest() && pred.contains(&Value::UInt(pos)) { - self.current = self.current.max(row.item_index + 1); + let finished = row.item_index; + self.current = self.current.max(finished + 1); + // A prefix release, which is sound because rows are emitted for the + // lowest offered position only: everything at or below the one that + // just finished has finished too. Releasing it is what makes the + // source's own release state this drive's progress record, so a + // replacement drive is offered what this one did not finish and + // nothing it did — see `src/ccl/design/live-update.md`, "The model: + // every carrier has its own cut". + self.source_producer + .release(TileGuard::Function(FunctionGuard::Domain( + Predicate::LessThanEq(Value::UInt(finished)), + ))); } self.window.compact(pred); self.debug_assert_window_invariants(); @@ -3409,6 +3446,7 @@ fn next_decided_position(tile: &Tile, pos: usize) -> Option { fn body_decision_at( tile: &Tile, pos: usize, + write_keys: &[Value], tap_fields: &[String], ) -> Option<(bool, Vec, Vec)> { let Tile::SealedFunction { @@ -3435,20 +3473,21 @@ fn body_decision_at( let Value::Record(payload) = *inner else { return None; }; - // The write set is the writes tuple `(_0, …, _{w-1})` in index order, followed - // by each reply tap's value — the order the caller's `write_keys` aligns with - // (carries then taps). + // The write set is keyed by the variable written, so it is read back in + // `write_keys` order — the order the caller aligns with (carries then taps). + // Each entry may itself be record-valued (a store holding a record). let mut writes = Vec::with_capacity(tap_fields.len()); match payload.get(F_WRITES)? { - // The normal case: the writes tuple is a record `{_0, …, _{w-1}}`. Each - // entry may itself be record-valued (a store holding a record). Value::Record(writes_rec) => { - for j in 0..writes_rec.len() { - writes.push(writes_rec.get(&tuple_field(j))?.clone()); + for key in write_keys.iter().take(writes_rec.len()) { + let Value::String(name) = key else { + return None; + }; + writes.push(writes_rec.get(name.as_str())?.clone()); } } - // A read-only transaction's empty writes tuple `()` lowers to a unit - // value (not a record): zero carry writes, only taps contribute. + // A read-only transaction's empty write set lowers to a unit value (not + // a record): zero carry writes, only taps contribute. Value::Unit => {} _ => return None, } @@ -3874,7 +3913,7 @@ impl TileProducer for TransactWriterProducer { && Some(pos) != self.last_decided_pos && let Some(frontier) = snapshot { - match body_decision_at(&body_tile, pos, &self.tap_fields) { + match body_decision_at(&body_tile, pos, &self.write_keys, &self.tap_fields) { // Grant: propose the write set; the operator decides whether it // commits — its ack releases the driver row, which is what advances // the driver past this item — or is stale, leaving the item to be @@ -4151,11 +4190,14 @@ mod tests { } } - /// The `commit` payload extent for a single-key writer: `{writes: {_0: value}}`. - fn commit_payload_extent() -> Extent { + /// The `commit` payload extent for a single-key writer: `{writes: {acc: value}}`. + /// + /// The write set is keyed by the variable written, so these fixtures name + /// the key the same way the CCL side does. + fn commit_payload_extent(key: &str) -> Extent { Extent::Record(HashMap::from([( F_WRITES.to_string(), - Extent::Record(HashMap::from([(tuple_field(0), value_extent())])), + Extent::Record(HashMap::from([(key.to_string(), value_extent())])), )])) } @@ -4169,15 +4211,11 @@ mod tests { ])) } - /// A `` `commit({writes: {_0, _1, …}}) `` decision value from its per-key write values. - fn commit_value(writes: Vec) -> Value { - let writes_rec = Value::Record( - writes - .into_iter() - .enumerate() - .map(|(i, v)| (tuple_field(i), v)) - .collect(), - ); + /// A `` `commit({writes: {key: write}}) `` decision value. The write set is + /// keyed by the variable written, so a fixture names its key the same way + /// the writer consuming the decision does. + fn commit_value(key: &str, write: Value) -> Value { + let writes_rec = Value::Record(HashMap::from([(key.to_string(), write)])); Value::Union { tag: FieldKey::Name(V_COMMIT.into()), inner: Box::new(Value::Record(HashMap::from([( @@ -4207,25 +4245,30 @@ mod tests { struct AddIfBody { input: Box, tiling: Tiling, + /// The accumulator this body writes. The write set is keyed by the + /// variable written, so a fixture has to name its key the same way the + /// writer that consumes the decision does. + key: String, /// The guard threshold: `commit` iff `item > threshold` (`i64::MIN` ⇒ an /// unconditional loop, `commit` everywhere). threshold: i64, } impl AddIfBody { - fn new(input: Box, threshold: i64) -> Self { + fn new(input: Box, threshold: i64, key: &str) -> Self { let tiling = Tiling::SealedFunction { domain: Extent::Base(BaseType::UInt), // Decision variant `` {`commit{{writes: {_0}}} | `abort} `` — a // `Scalar(Union)` codomain (commit=0, abort=1). codomain: Box::new(Tiling::Scalar(decision_union_extent( - commit_payload_extent(), + commit_payload_extent(key), ))), }; Self { input, tiling, threshold, + key: key.to_string(), } } } @@ -4244,6 +4287,7 @@ mod tests { self.input .subscribe(self.input.tiling().universal_guard(), consumer, scheduler); Box::new(AddIfBodyProducer { + key: self.key.clone(), base: ProducerBase::new(AddIfBodyProducer::alloc_id(), &self.tiling), input, threshold: self.threshold, @@ -4255,6 +4299,7 @@ mod tests { base: ProducerBase, input: Box, threshold: i64, + key: String, } impl TileProducer for AddIfBodyProducer { @@ -4283,7 +4328,7 @@ mod tests { panic!("AddIfBody prev/item are Ints"); }; rows.push(if i > self.threshold { - commit_value(vec![int(p + i)]) + commit_value(&self.key, int(p + i)) } else { abort_value() }); @@ -4292,7 +4337,7 @@ mod tests { domain, codomain: Box::new(Tile::Scalar(ColumnValue::from_values( rows, - &decision_union_extent(commit_payload_extent()), + &decision_union_extent(commit_payload_extent(&self.key)), ))), // A per-position decision map: the decision stream is final // exactly when its input is, as a compiled body's operator chain @@ -4322,6 +4367,8 @@ mod tests { Vec::new(), key_extent(), value_extent(), + // A store built with its source, not one resuming a running program. + 0, ); let set_body = store.body_input_setter(); let fan = Rc::new(FanOut::new_cyclic(Box::new(store))); @@ -4331,8 +4378,9 @@ mod tests { vec![acc.clone()], vec![value_extent()], value_extent(), + 0, ); - set_body(Box::new(AddIfBody::new(Box::new(driver), threshold))); + set_body(Box::new(AddIfBody::new(Box::new(driver), threshold, "acc"))); (fan, acc) } @@ -5425,7 +5473,7 @@ mod tests { // release intersection. Without it the intersection would be the // writer's ack alone, and a superseded row could not be reclaimed // before its item finished. - let body = AddIfBody::new(Box::new(Memo::new(driver_fan.branch())), i64::MIN); + let body = AddIfBody::new(Box::new(Memo::new(driver_fan.branch())), i64::MIN, "pool"); set_writer(Box::new(TransactWriter::new( store_fan.branch(), Box::new(body), diff --git a/src/interpreter/design-operators.md b/src/interpreter/design-operators.md index eacc03179..f3347fa6b 100644 --- a/src/interpreter/design-operators.md +++ b/src/interpreter/design-operators.md @@ -234,7 +234,7 @@ A single-writer induction store is the degenerate no-conflict case of this same ### The decision record -A writer body returns one **decision variant** per transaction, `` {`commit{𝑃} | `abort} `` (`ccl_utils::wrap_decision_variant`). `` `commit `` carries the payload record 𝑃 = `{writes, to_*}` — the positional tuple of proposed per-key new values, plus one field per reply tap — and `` `abort `` is the nullary whole-transaction deny: carry, no proposal. Making the grant/deny the *tag* rather than a `commit` field leaves "denied yet real writes" unrepresentable. `body_decision_at` decodes the tag by name, so the two ends agree without a canonical arm position. A tap fed under one arm of cross-key *routing* carries a companion `to__k__fire : Bool` gate holding that tap's own control-flow path (see [mutability.md](../ccl/design/mutability.md#general-in-transaction-conditionals-and-conditional-writes), "General in-transaction conditionals (and conditional writes)"). The grant path omits a non-fired tap from the commit delta, so a routed reply fires only on its own route. A tap whose path *is* the commit — a single-guard or spine feed — carries no gate and fires with its transaction, keeping unconditional programs at their gate-free shape. +A writer body returns one **decision variant** per transaction, `` {`commit{𝑃} | `abort} `` (`ccl_utils::wrap_decision_variant`). `` `commit `` carries the payload record 𝑃 = `{writes, to_*}` — the proposed new values keyed by the variable each is for, plus one field per reply tap — and `` `abort `` is the nullary whole-transaction deny: carry, no proposal. Making the grant/deny the *tag* rather than a `commit` field leaves "denied yet real writes" unrepresentable. `body_decision_at` decodes the tag by name, so the two ends agree without a canonical arm position. A tap fed under one arm of cross-key *routing* carries a companion `to__k__fire : Bool` gate holding that tap's own control-flow path (see [mutability.md](../ccl/design/mutability.md#general-in-transaction-conditionals-and-conditional-writes), "General in-transaction conditionals (and conditional writes)"). The grant path omits a non-fired tap from the commit delta, so a routed reply fires only on its own route. A tap whose path *is* the commit — a single-guard or spine feed — carries no gate and fires with its transaction, keeping unconditional programs at their gate-free shape. ### Convergence: the writer re-arms, one step per pull diff --git a/src/interpreter/http_server.rs b/src/interpreter/http_server.rs index 7daf0ec5c..67735b696 100644 --- a/src/interpreter/http_server.rs +++ b/src/interpreter/http_server.rs @@ -51,14 +51,26 @@ type RouteMap = Arc>>; /// which returns a `Receiver` delivering matching requests. Requests that do not match /// any registered route receive an immediate 404. /// -/// The background dispatcher thread is spawned once on construction and runs until the -/// server is dropped. +/// The background dispatcher thread is spawned once on construction and runs until this +/// handle is dropped, which is what releases the port. A program serves a port for as +/// long as some version of it binds a route there; dropping the last handle closes the +/// socket rather than leaving the address answering 404 forever (see +/// [`SourceSinkRegistry`](crate::ccl::context::SourceSinkRegistry)). pub struct SharedHttpServer { /// Routing table: `(method, path)` → sender half of each route's channel. /// /// Protected by a `Mutex` so that new routes can be registered after the /// dispatcher thread is already running. routes: RouteMap, + + /// The listener, shared with the dispatcher thread. + /// + /// Held here as well as there so that dropping this handle can end the + /// thread: `unblock` makes its `recv` return an error, which is the shutdown + /// the loop already handles by telling every route `None`. Moving the server + /// into the thread alone would leave the port bound for the life of the + /// process, since nothing else can reach it to stop the loop. + server: Arc, } impl SharedHttpServer { @@ -68,13 +80,15 @@ impl SharedHttpServer { /// returned immediately at construction rather than causing a silent hang /// after the program appears to start successfully. pub fn new(port: u16) -> Result> { - let server = Server::http(format!("0.0.0.0:{port}"))?; + let server = Arc::new(Server::http(format!("0.0.0.0:{port}"))?); debug!("HTTP server listening on 0.0.0.0:{port}"); let routes: RouteMap = Arc::new(Mutex::new(HashMap::new())); let routes_bg = routes.clone(); + let server_bg = server.clone(); thread::spawn(move || { + let server = server_bg; loop { match server.recv() { Ok(mut request) => { @@ -117,7 +131,7 @@ impl SharedHttpServer { } }); - Ok(Self { routes }) + Ok(Self { routes, server }) } /// Register a new `(method, path)` route and return the `Receiver` for incoming requests. @@ -133,6 +147,117 @@ impl SharedHttpServer { self.routes.lock().unwrap().insert((method, path), tx); rx } + + /// Stop dispatching `method path`, so requests to it get the dispatcher's + /// 404 rather than being buffered for a reader that no longer exists. + /// + /// A route outlives the version of the program that opened it — the listener + /// and its table entry are held by the source/sink registry, not by the + /// operator graph — so a version that stops serving one has to say so. + /// Without this the route still matches, still buffers, and the client waits + /// on a reply nobody will compute. + pub fn unregister(&self, method: &str, path: &str) { + self.routes + .lock() + .unwrap() + .remove(&(method.to_string(), path.to_string())); + } +} + +/// An `http_serve` route a compilation named but did not open. +/// +/// A compile that only answers a question — `/diff`, and the planned tree the +/// state guard reads — runs against the endpoints the program already holds. A +/// version it is asked about may name a route the program does not serve, and +/// opening one would make asking change what the program does: it binds a socket +/// and registers a route that outlive the throwaway context, so the address +/// starts answering for a version nobody installed and the real compile then +/// fails to bind the port it just took. +/// +/// So such a route lowers to this instead. It answers the type questions lowering +/// and inference put to a source and nothing else, which is all a compile that +/// stops above operator conversion asks. Reaching a runtime method means one +/// escaped into an operator graph, so each is [`unreachable`]. +pub struct UnopenedRoute { + id: String, +} + +impl UnopenedRoute { + pub fn new(id: String) -> Self { + Self { id } + } +} + +impl DataSourceDomainExtentImpl for UnopenedRoute { + fn get_id(&self) -> &str { + &self.id + } + + fn element_extent(&self) -> Extent { + Extent::Base(BaseType::UInt) + } + + fn output_value_extent(&self) -> Extent { + Extent::Base(BaseType::String) + } + + fn output_type(&self) -> Type { + Type::Base(BaseType::String) + } + + fn check_for_new_data(&mut self) -> bool { + unreachable!("`{}` was never opened, so nothing drives it", self.id) + } + + fn get_yield_predicate(&self) -> Predicate { + unreachable!("`{}` was never opened, so it yields nothing", self.id) + } + + fn get_elements(&self, _producer: &str) -> ColumnValue { + unreachable!("`{}` was never opened, so it holds no elements", self.id) + } + + fn get(&self, _keys: ColumnValue) -> ColumnValue { + unreachable!("`{}` was never opened, so it answers no key", self.id) + } + + fn release(&mut self, _producer: &str, _obsolete: Predicate) { + unreachable!( + "`{}` was never opened, so nothing subscribed to it", + self.id + ) + } + + fn carry_release_to_new_producers(&mut self) { + unreachable!( + "`{}` was never opened, so it has no release to carry", + self.id + ) + } + + fn first_position_for_a_new_producer(&self) -> usize { + unreachable!("`{}` was never opened, so it offers no position", self.id) + } +} + +/// The reply sink of an [`UnopenedRoute`]. Dispatching to it would mean a version +/// nobody installed answering a request. +pub struct UnopenedRouteSink; + +impl DataSink for UnopenedRouteSink { + fn process(&self, _tile: &Tile) { + unreachable!("a route that was never opened has no client to answer"); + } +} + +impl Drop for SharedHttpServer { + fn drop(&mut self) { + // Ends the dispatcher thread, which is what drops its `Arc` and + // so closes the listener. `recv` returns an error, and the loop's error + // arm already sends `None` to every remaining route — the shutdown signal + // [`register`] documents. + self.server.unblock(); + } } // --------------------------------------------------------------------------- @@ -432,4 +557,12 @@ impl DataSourceDomainExtentImpl for HttpServerDataSource { fn release(&mut self, producer: &str, obsolete: Predicate) { self.buf.release(producer, obsolete); } + + fn carry_release_to_new_producers(&mut self) { + self.buf.releases.carry_to_new_producers(); + } + + fn first_position_for_a_new_producer(&self) -> usize { + self.buf.first_index_for_a_new_producer() + } } diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index ba12839e2..eef6cb40d 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -7,6 +7,7 @@ mod binop; pub mod commit_operator; pub mod http_server; pub mod operator_conversion; +mod producer_releases; mod scheduler; pub mod sinks; mod stdio; diff --git a/src/interpreter/operator_conversion.rs b/src/interpreter/operator_conversion.rs index 143118be7..46ef4ab46 100644 --- a/src/interpreter/operator_conversion.rs +++ b/src/interpreter/operator_conversion.rs @@ -3,7 +3,11 @@ use log::trace; use crate::{ ccl::{ AggregateKind, Builtin, Expr, F_WRITES, FieldKey, Lit, Name, ProjKey, TagMap, TransactKey, - Type, TypedExprNode, V_COMMIT, WriterSite, ccl_utils::is_trivially_true_predicate, + Type, TypedExprNode, V_COMMIT, WriterSite, + ccl_utils::strip_refinements, + ccl_utils::{free_names, is_trivially_true_predicate}, + content_hash::{ContentHash, resolved_hash}, + provenance::NodeId, symbolic::symbolic, }, interpreter::{ @@ -24,6 +28,7 @@ use crate::{ commit_operator::{ AsOf, AsOfField, CommitOperator, InductionDriver, InductionStore, StoreDenseRead, StoreFinalRead, StoreValueStream, TransactDriver, TransactWriter as CommitWriter, + store_frontier, store_value_at, }, tile_operators::{ Aggregate, Constant, Converse, ExtractAggregate, ExtractFinal, FanOut, Filter, @@ -36,7 +41,11 @@ use crate::{ }, util::ScopeStack, }; -use std::{cell::RefCell, collections::HashMap, rc::Rc}; +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + rc::Rc, +}; /// Converts a λ-eliminated CCL expression into an operator graph. /// @@ -127,16 +136,13 @@ pub fn convert_record_fields_to_operators( domain, } = &bound_expr.node { - let info = build_transact_store(keys, writers, domain, ctx)?; - ctx.register_store(binding.name.clone(), info); + ctx.bind_store(&binding.name, bound_expr, keys, writers, domain)?; return convert_record_fields_to_operators(body, ctx); } - let bound_op = convert_impl(bound_expr, None, ctx)?; - let fan_out = Rc::new(FanOut::new(Box::new(Memo::new(bound_op)))); let mut scope = ctx.enter_scope(); // No surrounding iteration here (this entry point compiles a // Let* Record* chain from the top); every binding is free. - scope.bind(&binding.name, fan_out, BindingKind::Free); + scope.bind_let(&binding.name, bound_expr, None)?; convert_record_fields_to_operators(body, &mut scope) } TypedExprNode::Record(fields) => fields @@ -243,34 +249,60 @@ enum StoreReadKind { /// changelog: a read is a [`StoreDenseRead`] folding the changelog at every /// position of the loop extent (`StoreReadInfo::induction_extent`) into the /// dense history `D ⇀ V` — the changelog counterpart of `Induction`'s - /// `.writes.(index)`, serving both scalar-final and co-iterated reads. + /// `__hist.k`, serving both scalar-final and co-iterated reads. InductionChangelog, } /// How to read one key (variable) of a transactional store. The scalar-read /// reduction to the current/final value (`final_or_default` → `ExtractFinal`) is /// expressed in the CCL, not here. +#[derive(Clone)] struct KeyReadInfo { + /// What identifies this variable across versions, or `None` for a key that is + /// not a variable. + /// + /// `Some` exactly for a mutable variable, so this is also what + /// [`carry_forward`](Self::carry_forward) answers: a reply tap is a + /// per-commit event, and a key that carries no value between ticks has no + /// state to carry between versions either. One field rather than two that + /// would have to agree. + /// + /// Distinct from `runtime_key`, which labels the variable inside *this* + /// store's record and is the user's spelling alone: a spelling is unique per + /// record, and an identity has to be unique per program. Assigned by the walk + /// the guard also reads ([`mutable_variable_paths`]). + carried: Option, /// The runtime key the variable's value lives under in the commit store map /// (`commit` stores only; `Value::Unit` for induction stores). runtime_key: Value, /// The per-commit value extent for [`StoreValueStream`] (`commit` stores /// only; the accumulator value extent for induction stores). value_extent: Extent, - /// The key's position in the writer's `writes` tuple: `__hist.k` projects - /// `.writes.(index)` off the store body stream (`Induction` stores). - index: usize, +} + +impl KeyReadInfo { /// Whether the key's value carries forward across commit ticks that don't - /// write it (`commit` stores): `true` for a mutable variable (persistent value), - /// `false` for a reply tap (a per-commit event). See - /// [`StoreValueStream::carry_forward`]. - carry_forward: bool, + /// write it (`commit` stores). See [`StoreValueStream::carry_forward`]. + fn carry_forward(&self) -> bool { + self.carried.is_some() + } +} + +impl StoreReadInfo { + /// This store's keys that carry state, each with the identity it carries it + /// under. + fn carried_keys(&self) -> impl Iterator { + self.keys + .values() + .filter_map(|k| k.carried.as_ref().map(|path| (path, k))) + } } /// A built transactional store, registered under its `__hist` binder so each /// per-variable read (`__hist.k`) can branch the shared fan and project key /// `k`. The scalar-read reduction (`final_or_default` → `ExtractFinal`) is /// expressed in the CCL, not here. +#[derive(Clone)] struct StoreReadInfo { /// The cyclic store fan — a [`FanOut`] over the store body stream; every /// read is a branch of this one fan. @@ -283,6 +315,22 @@ struct StoreReadInfo { /// read enumerates (its [`StoreDenseRead`] trigger). `None` for a `commit` or /// dense `Induction` store. induction_extent: Option, + /// The sequence this store counts its positions in, which is what decides + /// whether a replacement may resume at this store's frontier. `None` for a + /// commit store, whose clock restarts with the store that counts it. + /// + /// Its variables' *values* are not held here: a store's value rides its own + /// fan as a [`Tile::Store`], so a version replacing this one reads them off + /// [`FanOut::cached_tile`] rather than through a second channel out of the + /// operator. + sequence: Option, + /// The token a term reading this store hashes to — the same role + /// [`LetBinding::correspondent`] plays, for a binding that lives in + /// [`transactional_stores`](OpConversionContext::transactional_stores) + /// rather than in the scope stack. Set by + /// [`bind_store`](OpConversionContext::bind_store), which is what knows the + /// store's identity. + correspondent: u64, } /// Compilation context for tile compilation. @@ -295,9 +343,35 @@ pub struct OpConversionContext { /// Variable bindings in scope, innermost scope last. Each binding /// carries a [`BindingKind`] so [`TypedExprNode::Var`] lookups can /// dispatch on it without inspecting tile-level types. - scopes: ScopeStack, BindingKind)>, + scopes: ScopeStack, + /// What the version this one replaces left behind, to be adopted where the + /// computation is unchanged. Consulted at every `Let` binding + /// ([`bind_let`](Self::bind_let)) and every store ([`bind_store`](Self::bind_store)); + /// empty for a program's first compilation. + inherited: Inheritance, + /// What this compilation binds, for the version that replaces it. + minted: Inheritance, + /// The bindings this compilation built rather than adopted, by binder name. + /// + /// What makes reuse hereditary. A term is eligible for reuse only when none + /// of the names free in it is here, so an adopted operator is never left + /// reading a subgraph this compilation rebuilt. Checking the names rather + /// than folding "was it rebuilt" into the correspondent is what lets the + /// correspondent stay stable across compilations. + /// + /// A binding is in here whenever its operator is new, which covers more than + /// an edited term: a binding under an iteration is always rebuilt, and so is + /// one the previous version did not have. + rebuilt: HashSet, + /// How much of the previous version this compilation adopted. + reuse: ReuseTally, /// Maps source names to their runtime [`DataSourceDomainExtentImpl`]. sources: HashMap>>, + /// Every mutable variable's identity, by the `Transact` node that declares + /// it, from [`mutable_variable_paths`] over the tree about to be converted. + /// Installed by [`set_var_paths`](Self::set_var_paths) before conversion + /// begins; empty for a context converting no `Transact`. + var_paths: HashMap>, /// Transactional stores in scope, keyed by their `__hist` binder. A /// `let __hist = Transact{…}` builds the shared store once and registers /// it here; each variable read `__hist.k` projects key `k` off the shared @@ -306,6 +380,86 @@ pub struct OpConversionContext { transactional_stores: HashMap, } +/// What one compilation hands the version that replaces it: the operator behind +/// each `Let` binding it bound and each store it built, by the identity of the +/// term realized, and the value each mutable variable was holding. +/// +/// One type for both sides of the handover — what a compilation accumulates and +/// what it inherits are the same three things. +#[derive(Default)] +pub struct Inheritance { + operators: HashMap>, + stores: HashMap, + /// What each mutable variable hands to the variable that replaces it, by + /// identity. Read off `stores` at handover + /// ([`OpConversionContext::into_inheritance`]), so the copy a compilation is + /// still accumulating into carries none — only the one it hands on. + mutable_state: HashMap, +} + +/// What one mutable variable hands to the variable rebuilt in its place. +/// +/// The value and the position travel together because they only mean anything +/// together: a value is what the recurrence held at the position it had reached, +/// so seeding from one position and resuming at another either decides a position +/// twice or skips it. +pub(crate) struct CarriedState { + /// The value the variable held where its recurrence had reached. + value: Value, + /// Where that was, or `None` for a recurrence with no position to hand on: a + /// transaction's commit clock restarts with the store that counts it, and the + /// replacement seeds tick 0 from `value`. + resumption: Option, +} + +/// Where a recurrence had reached, for the recurrence rebuilt in its place. +/// +/// A property of the store, carried per variable because a variable is what has +/// an identity — a store has none. Every variable of one store carries the same +/// resumption, and the store rebuilt in their place reads it back off any +/// variable it declares. +#[derive(Clone)] +pub(crate) struct Resumption { + /// The first position the replacement decides — the retired store's frontier, + /// which is the position it had reached and not yet decided. + /// + /// Taken from the retired store's own frontier rather than from how far its + /// source has been released. A drive retains the input it reads one position + /// back through, so the source still owes the replacement an element the + /// recurrence has already decided — reading the resume position off the + /// source would decide it a second time. + position: usize, + /// What `position` counts in. A replacement counting in another sequence + /// starts its own count. + sequence: Sequence, +} + +/// How much of the previous version a compilation adopted. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ReuseTally { + /// Bindings and stores taken from the previous version rather than built. + pub adopted: usize, + /// Bindings and stores bound in total. + pub bound: usize, +} + +/// One `Let` binding in scope during conversion. +pub(crate) struct LetBinding { + /// The fan-out every use of the binding branches off. + pub(crate) fan: Rc, + /// Whether the binding was compiled inside an iteration scope. + pub(crate) kind: BindingKind, + /// The token a term free in this binding hashes to — the identity hash of + /// the term this binding computes, whether or not its operator was rebuilt. + /// + /// Naming a binding by what it computes is what makes two compilations of + /// one source agree, so an unchanged program reuses on its first update + /// rather than settling in over several. Whether the operator behind the + /// binding is still the previous version's is a separate question, answered + /// by [`rebuilt`](OpConversionContext::rebuilt). + correspondent: u64, +} + /// RAII scope guard for [`TileCompileContext`]. /// /// Created by [`TileCompileContext::enter_scope`]; pops the innermost scope when @@ -340,6 +494,15 @@ impl OpConversionContext { Self::default() } + /// Install the identities the tree about to be converted declares. + /// + /// Read off the tree once, before conversion, so a store is built under the + /// identity the guard checked it against rather than under a second + /// derivation of one. + pub fn set_var_paths(&mut self, expr: &Expr) { + self.var_paths = mutable_variable_paths(expr); + } + /// Register a data-source implementation under `name`. /// /// After registration, [`Type::DataSource`] resolves to @@ -373,24 +536,309 @@ impl OpConversionContext { TileCompileContextGuard { ctx: self } } - /// Bind `name` to `binding` in the innermost scope. + /// Look up `name` from innermost scope outward. + pub(crate) fn lookup(&self, name: &Name) -> Option<&LetBinding> { + self.scopes.lookup(name) + } + + /// Whether an operator for `bound_expr` may be adopted from the previous + /// version — that is, whether every binding it reads is still that version's. /// - /// `kind` records whether the binding was compiled inside an iteration - /// scope ([`BindingKind::Aligned`]) or outside one ([`BindingKind::Free`]); - /// [`Self::lookup`] returns this alongside the [`FanOut`] so the - /// [`TypedExprNode::Var`] arm can dispatch without inspecting tile types. - pub(crate) fn bind(&mut self, name: &Name, binding: Rc, kind: BindingKind) { - self.scopes.bind(name.clone(), (binding, kind)); + /// See [`rebuilt`](Self::rebuilt). Bindings are bound in dependency order, so + /// checking the names free in one term is transitive: a binding that reads a + /// rebuilt one is itself recorded as rebuilt. + fn reads_only_adopted(&self, bound_expr: &Expr) -> bool { + self.rebuilt.is_empty() + || free_names(bound_expr) + .iter() + .all(|name| !self.rebuilt.contains(name)) } - /// Look up `name` from innermost scope outward. - pub(crate) fn lookup(&self, name: &Name) -> Option<&(Rc, BindingKind)> { - self.scopes.lookup(name) + /// The correspondent of every binding in scope, innermost last — the scope + /// [`resolved_hash`] resolves a term's free variables against. + /// + /// Stores come first, so a `Let` binder shadowing a store name resolves to + /// the binder. Their names are α-unique, so the two sets are disjoint in + /// practice and the order only fixes a tie that cannot arise. + fn binder_correspondents(&self) -> Vec<(&Name, u64)> { + self.transactional_stores + .iter() + .map(|(name, info)| (name, info.correspondent)) + .chain( + self.scopes + .iter_bindings() + .map(|(name, binding)| (name, binding.correspondent)), + ) + .collect() } - /// Register a built transactional store under its `__hist` binder. - fn register_store(&mut self, name: Name, info: StoreReadInfo) { - self.transactional_stores.insert(name, info); + /// Build the store `bound_expr` describes, or adopt the one a previous + /// version built for the same `Transact` term, and register it under `name`. + /// + /// The store-shaped counterpart of [`bind_let`](Self::bind_let), and reuse + /// matters more here than anywhere else: the store *is* the program's + /// mutable state, so adopting one is what carries an accumulator across an + /// update. Its correspondent is the term's identity whether the store was + /// adopted or rebuilt, exactly as [`LetBinding::correspondent`] is; a term + /// reading a rebuilt store is kept from adoption by + /// [`rebuilt`](Self::rebuilt), not by its correspondent changing. + fn bind_store( + &mut self, + name: &Name, + bound_expr: &Expr, + keys: &[TransactKey], + writers: &[WriterSite], + domain: &Type, + ) -> Result<(), ConversionError> { + let identity = resolved_hash(bound_expr, &self.binder_correspondents()); + self.reuse.bound += 1; + // The identities the guard checked this version against, for the store + // about to be built. Assigned by one walk of the planned tree + // ([`mutable_variable_paths`]) rather than derived here, so a variable is + // built under the identity it was checked under. + let paths = self + .var_paths + .get(&bound_expr.node_id()) + .cloned() + .unwrap_or_default(); + debug_assert_eq!( + paths.len(), + keys.len(), + "the identity walk and conversion disagree about how many variables this store declares" + ); + + let adoptable = self + .reads_only_adopted(bound_expr) + .then(|| self.inherited.stores.get(&identity).cloned()) + .flatten(); + let info = match adoptable { + Some(info) => { + self.reuse.adopted += 1; + trace!("reusing store for binding {name}"); + info + } + None => { + trace!("building store for binding {name}"); + self.rebuilt.insert(name.clone()); + build_transact_store(keys, writers, domain, &paths, self)? + } + }; + let info = StoreReadInfo { + correspondent: identity.0, + ..info + }; + self.minted.stores.insert(identity, info.clone()); + self.transactional_stores.insert(name.clone(), info); + Ok(()) + } + + /// Compile `bound_expr` into the fan-out its uses branch off, reusing the + /// operator a previous version built for the same computation where there is + /// one, and bind it to `name`. + /// + /// A `Let` is the reuse boundary because it is already the sharing boundary: + /// the fan-out and [`Memo`] a binding compiles to are what let several uses + /// draw on one operator, and a new version's use is just one more. A late + /// branch does not re-subscribe upstream — it pulls the same `MemoProducer`, + /// so a reused operator hands the new version whatever it still holds. + /// + /// *Whatever it still holds*, not everything it ever produced: a `Memo` drops + /// what its consumers release, so what a new version inherits is bounded by + /// what the retired one had finished with. The new subscriber is told as much + /// — its release guard starts at what the fan-out has already released — and a + /// binding released in full is not offered at all + /// ([`FanOut::released_in_full`]). + /// + /// Reuse is declined for a binding compiled under an iteration + /// ([`BindingKind::Aligned`]). Such an operator is parameterized by the + /// iteration input threaded into it, which is not part of the term and so + /// not part of its identity; reusing it would keep the input the previous + /// version supplied. + fn bind_let( + &mut self, + name: &Name, + bound_expr: &Expr, + bound_input: Option>, + ) -> Result<(), ConversionError> { + let kind = if bound_input.is_some() { + BindingKind::Aligned + } else { + BindingKind::Free + }; + // The correspondent names what the binding computes, so it is taken for every + // binding — including an `Aligned` one, whose operator is never adopted + // but whose readers still need a stable name for it. + let identity = resolved_hash(bound_expr, &self.binder_correspondents()); + + self.reuse.bound += 1; + let adoptable = (kind == BindingKind::Free && self.reads_only_adopted(bound_expr)) + .then(|| self.inherited.operators.get(&identity).cloned()) + .flatten(); + if let Some(fan) = adoptable { + self.reuse.adopted += 1; + trace!("reusing operator for binding {name}"); + self.minted.operators.insert(identity, fan.clone()); + self.scopes.bind( + name.clone(), + LetBinding { + fan, + kind, + correspondent: identity.0, + }, + ); + return Ok(()); + } + + // `bound_expr` is compiled unconditionally — whether or not the body + // references the binding. This is why `planning` must make every + // function-typed bound expr iteration-bearing: an unused, + // non-iteration-bearing function-typed binding would otherwise reach an + // `input=None` arm here and error. It also means a dead iterable + // binding is materialised rather than dropped; #232 tracks making + // iteration use-driven (lazy `Let` compilation / DCE) so this eager + // compile is no longer forced. + trace!("building operator for binding {name}"); + self.rebuilt.insert(name.clone()); + let bound_op = convert_impl(bound_expr, bound_input, self)?; + let fan = Rc::new(FanOut::new(Box::new(Memo::new(bound_op)))); + // An `Aligned` operator is offered to the next version even though it + // will not be adopted: the offer is keyed by the term, and declining is + // the reader's decision, made at the point of adoption. + self.minted.operators.insert(identity, fan.clone()); + self.scopes.bind( + name.clone(), + LetBinding { + fan, + kind, + correspondent: identity.0, + }, + ); + Ok(()) + } + + /// Every variable the running program holds that `planned` cannot take over. + /// + /// Checked before anything is torn down, so a version that would lose a + /// value or change its type is refused while the running program is whole. + /// A type change is refused rather than reseeded because the value would + /// become the seed of a store built for another shape, and the store fails + /// on its first pull rather than at the swap. + pub fn state_conflicts(&self, planned: &Expr) -> Vec { + let declared = declared_state(planned); + let mut out = Vec::new(); + for info in self.minted.stores.values() { + for (path, key) in info.carried_keys() { + let Some(decl) = declared.get(path) else { + out.push(StateConflict::Dropped { path: path.clone() }); + continue; + }; + // A declared type the conversion context cannot resolve to an + // extent is a compile error the real compile will raise with its + // own diagnostic; this check has nothing to add. + let Ok(extent) = self.extent_of(&decl.ty) else { + continue; + }; + if extent != key.value_extent { + out.push(StateConflict::Retyped { + path: path.clone(), + held: key.value_extent.clone(), + declared: extent, + }); + } + } + } + out + } + + /// The value each mutable variable currently holds, by the identity state is + /// carried under. Readable before the version is retired, so a replacement + /// can be checked against what it would inherit. + /// + /// A store [`state_conflicts`](Self::state_conflicts) treated as carryable is + /// skipped here only when it holds nothing, because a store with a value that + /// this drops is the one outcome the guard exists to prevent: the replacement + /// reseeds from the init, the program carries on answering, and only the + /// history is gone. So the absent cached tile is an assertion — a store's fan + /// is always cyclic ([`FanOut::new_cyclic`]) and a cyclic fan always has one — + /// while the two ways of holding nothing are ordinary cases: a store that has + /// decided no position has no frontier, and a variable no position has written + /// has no value at the frontier. Either way its replacement reads its init, + /// which is what it would have read anyway. + pub(crate) fn live_state(&self) -> HashMap { + let mut out: HashMap = HashMap::new(); + for info in self.minted.stores.values() { + let Some(tile) = info.fan.cached_tile() else { + debug_assert!( + false, + "a store's fan is cyclic, so it has a cached tile: {:?}", + info.keys.keys().collect::>() + ); + continue; + }; + // No frontier means no position has been decided, so there is no + // accumulated value to hand over and nothing is lost by starting the + // replacement at its init. + let Some(frontier) = store_frontier(&tile) else { + continue; + }; + for (path, key) in info.carried_keys() { + if let Some(value) = store_value_at(&tile, frontier, &key.runtime_key) { + let prior = out.insert( + path.clone(), + CarriedState { + value, + resumption: info.sequence.clone().map(|sequence| Resumption { + position: frontier, + sequence, + }), + }, + ); + debug_assert!( + prior.is_none(), + "{path} is declared by two stores, so the walk that assigns \ +identities is not distinguishing them", + ); + } + } + } + out + } + + /// Everything a replacement may adopt from this version: the bindings and + /// stores it minted, and the value each mutable variable holds. + /// + /// Reopens every fan first. A fan closes when its subscribers go, and this + /// version's are about to; a carried-forward one has to be open for the + /// replacement to subscribe to it. + pub fn into_inheritance(mut self) -> Inheritance { + for fan in self.minted.operators.values() { + fan.reopen(); + } + for info in self.minted.stores.values() { + info.fan.reopen(); + } + self.minted.mutable_state = self.live_state(); + // Read after `live_state`, which is what takes a store's value off its + // fan; a store released in full still hands its variables on, and it is + // only the operator that is withdrawn. + self.minted + .operators + .retain(|_, fan| !fan.released_in_full()); + self.minted + .stores + .retain(|_, info| !info.fan.released_in_full()); + self.minted + } + + /// Seed this context with what a previous version bound. + pub fn inherit(&mut self, inheritance: Inheritance) { + self.inherited = inheritance; + } + + /// How much of the previous version this compilation adopted. `0` adopted + /// for a first compilation, which inherits nothing. + pub fn reuse(&self) -> ReuseTally { + self.reuse } /// Look up a transactional store by its `__hist` binder. @@ -567,8 +1015,7 @@ fn convert_impl_inner( domain, } = &bound_expr.node { - let info = build_transact_store(keys, writers, domain, ctx)?; - ctx.register_store(binding.name.clone(), info); + ctx.bind_store(&binding.name, bound_expr, keys, writers, domain)?; return convert_impl(body, input, ctx); } let (bound_input, body_input) = match input { @@ -585,23 +1032,8 @@ fn convert_impl_inner( // than re-apply via `MapResult`. A free binding (no input) // is a standalone function; references under an iteration must // wrap it in `MapResult` to look up at each position. - let kind = if bound_input.is_some() { - BindingKind::Aligned - } else { - BindingKind::Free - }; - // `bound_expr` is compiled unconditionally — whether or not - // `body` references the binding. This is why `planning` must - // make every function-typed bound expr iteration-bearing: an - // unused, non-iteration-bearing function-typed binding would - // otherwise reach an `input=None` arm here and error. It also - // means a dead iterable binding is materialised rather than - // dropped; #232 tracks making iteration use-driven (lazy `Let` - // compilation / DCE) so this eager compile is no longer forced. - let bound_op = convert_impl(bound_expr, bound_input, ctx)?; - let fan_out = Rc::new(FanOut::new(Box::new(Memo::new(bound_op)))); let mut scope = ctx.enter_scope(); - scope.bind(&binding.name, fan_out, kind); + scope.bind_let(&binding.name, bound_expr, bound_input)?; convert_impl(body, body_input, &mut scope) } @@ -1112,9 +1544,9 @@ fn convert_impl_inner( } TypedExprNode::Var(name) => { - if let Some((fan_out, kind)) = ctx.lookup(name) { - let kind = *kind; - let op = fan_out.branch(); + if let Some(binding) = ctx.lookup(name) { + let kind = binding.kind; + let op = binding.fan.branch(); // Aligned bindings already vary in lockstep with the // surrounding iteration — return the FanOut branch directly. // Free bindings are standalone functions; under an @@ -1475,12 +1907,13 @@ fn build_transact_store( keys: &[TransactKey], writers: &[WriterSite], domain: &Type, + paths: &[VarPath], ctx: &mut OpConversionContext, ) -> Result { if !matches!(domain, Type::Txn) { - return build_induction_store(keys, writers, ctx); + return build_induction_store(keys, writers, domain, paths, ctx); } - build_commit_store(keys, writers, ctx) + build_commit_store(keys, writers, paths, ctx) } /// The reply taps on a writer body's `` {`commit{writes, to_*} | `abort} `` @@ -1522,6 +1955,7 @@ fn body_tap_fields(body_ty: &Type) -> Vec<(String, Type)> { fn build_commit_store( keys: &[TransactKey], writers: &[WriterSite], + paths: &[VarPath], ctx: &mut OpConversionContext, ) -> Result { // Each variable becomes a key under its `field_key`'s runtime value; the @@ -1542,23 +1976,37 @@ fn build_commit_store( // homogeneous store collapses the union to its single extent (the common // case, unchanged). let mut value_extents: Vec = Vec::new(); - for k in keys { + for (i, k) in keys.iter().enumerate() { let field = k.name.field_key(); let runtime_key = Value::String(field.clone().into()); let key_value_extent = ctx.extent_of(&k.init.ty)?; if !value_extents.contains(&key_value_extent) { value_extents.push(key_value_extent.clone()); } - // Seed tick 0 from the key's (literal or computed) init op. - let init_op = convert_impl(&k.init, None, ctx)?; + // Seed tick 0 from the value the retired version left this variable + // holding, or from the key's (literal or computed) init op when this + // version introduces it. Rebuilding a commit store therefore changes how + // a transaction decides without discarding what it has committed — the + // same rule the induction path follows, and the reason state is keyed by + // variable rather than by store. + let carried = ctx.inherited.mutable_state.get(&paths[i]); + let init_op: Box = match carried { + Some(carried) => { + trace!("resuming transactional {field} from the retired version's value"); + Box::new(Constant::new( + carried.value.clone(), + key_value_extent.clone(), + )) + } + None => convert_impl(&k.init, None, ctx)?, + }; init_ops.push((runtime_key.clone(), init_op)); let prior = keys_map.insert( field, KeyReadInfo { + carried: Some(paths[i].clone()), runtime_key, value_extent: key_value_extent, - index: 0, // unused for `commit` reads (keyed by `runtime_key`) - carry_forward: true, // mutable variable: value persists across commits }, ); debug_assert!( @@ -1655,13 +2103,14 @@ fn build_commit_store( let prior = keys_map.insert( field.clone(), KeyReadInfo { - runtime_key: Value::String(field.clone().into()), - value_extent: tap_value_extent, - index: 0, // unused for `commit` reads (keyed by `runtime_key`) // A reply tap is a per-commit event, not a persistent value: // emit it only at the tick that wrote it, so two writers' - // taps to one defer don't smear across the shared clock. - carry_forward: false, + // taps to one defer don't smear across the shared clock. It + // carries nothing between ticks and so nothing between + // versions. + carried: None, + runtime_key: Value::String(field.clone().into()), + value_extent: tap_value_extent, }, ); // A tap shares this map with the mutable-variable keys, so its @@ -1691,6 +2140,9 @@ fn build_commit_store( keys: keys_map, kind: StoreReadKind::Commit, induction_extent: None, + // Set by `bind_store`, which is what knows the store's identity. + correspondent: 0, + sequence: None, }) } @@ -1706,6 +2158,196 @@ fn induction_extent_is_positional(extent: &Extent) -> bool { matches!(extent, Extent::UIntRange(_) | Extent::DataSourceDomain(_)) } +/// A mutable variable's identity across versions of a program. +/// +/// The spelling carries the meaning and the index only disambiguates, because a +/// spelling is not unique on its own: a declaration can be shadowed, and a +/// function holding a whole stateful loop declares one variable per call site +/// once inlining has cloned its body. Neither of those has a name of its own to +/// borrow, so the index is what tells them apart. +/// +/// Counted among the variables that *share* the spelling rather than among all of +/// them, which is what makes it stable: a stateful loop added anywhere shifts +/// nothing unless it declares this same name. A version that does insert another +/// `total` between two existing ones renumbers them, and the variables below the +/// insertion read as dropped rather than as each other. +/// +/// This is deliberately not derived from anything the program computes. Two +/// instantiations of one function can differ *only* in their writer bodies once +/// arguments are substituted, so any content-derived identity either fails to +/// tell them apart or changes under exactly the edit state has to survive. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VarPath { + /// The variable's own spelling — [`Name::field_key`]. + name: String, + /// Its position among the variables of this spelling, in tree order. + index: usize, +} + +impl std::fmt::Display for VarPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The bare spelling while it is the only one, because that is what an + // author recognizes; the index only when it is doing work. + if self.index == 0 { + write!(f, "`{}`", self.name) + } else { + write!(f, "`{}` (#{})", self.name, self.index) + } + } +} + +/// The first position the source `domain` names will offer a producer registering +/// now, or `0` for a domain that is not a source. +fn source_start(domain: &Type, ctx: &OpConversionContext) -> usize { + let Type::DataSource(name) = domain else { + return 0; + }; + ctx.sources.get(name).map_or(0, |source| { + source.borrow().first_position_for_a_new_producer() + }) +} + +/// What a recurrence counts its positions in, named so that two versions can tell +/// whether they are counting in the same one. +/// +/// A position means nothing on its own: `3` is the fourth request a source +/// delivered or the fourth element of a list, so a replacement resumes at its +/// predecessor's frontier only when the two count in the same sequence, and starts +/// its own count otherwise. +/// +/// A source is named by itself, since it outlives every version reading it. A +/// collection is part of the program, so it is named by the term that computes it: +/// `[0, 2]` is the extent of `["y", "z"]` and of `["p", "q"]` alike, and resuming +/// the second fold at the first's frontier would skip elements nothing ever read. +#[derive(Clone, PartialEq, Eq, Debug)] +enum Sequence { + /// A data source's stream, by source name. + Source(String), + /// A collection this version computes, by the identity of the term. + Collection(ContentHash), +} + +/// The sequence a store over `domain` counts in, naming a collection by the +/// identity of `source`, the term the loop iterates. +fn sequence_of(domain: &Type, source: &Expr, ctx: &OpConversionContext) -> Sequence { + match domain { + Type::DataSource(name) => Sequence::Source(name.to_string()), + _ => Sequence::Collection(resolved_hash(source, &ctx.binder_correspondents())), + } +} + +/// Every mutable variable `expr` declares, by identity, with the type it is +/// declared at. +/// +/// One walk, shared by the guard and by conversion, so the identity a version is +/// checked against is the identity its stores are built under. Deriving it twice +/// would mean two computations that have to agree, and a disagreement reads as a +/// variable that is still declared having been dropped. +pub fn declared_state(expr: &Expr) -> HashMap { + let mut out = HashMap::new(); + for v in mutable_variables(expr) { + out.insert( + v.path, + DeclaredVariable { + ty: v.key.init.ty.clone(), + }, + ); + } + out +} + +/// What a version declares a mutable variable as — enough to decide whether the +/// running program's value can be seeded into it. +pub struct DeclaredVariable { + ty: Type, +} + +/// Every mutable variable's identity, in key order, by the `Transact` node that +/// declares it. +/// +/// Handed to conversion so that a store is built under the identities the guard +/// checked, rather than under a second derivation of them. +pub fn mutable_variable_paths(expr: &Expr) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + for v in mutable_variables(expr) { + out.entry(v.declared_by).or_default().push(v.path); + } + out +} + +/// One mutable variable as the identity walk sees it. +struct MutableVariable<'e> { + path: VarPath, + key: &'e TransactKey, + /// The `Transact` node declaring it, so conversion can ask for the identities + /// of the store it is building. + declared_by: NodeId, +} + +/// Every mutable variable in `expr`, in tree order, with its identity. +/// +/// The one place identities are assigned. `Transact` is the only node that +/// declares a mutable variable, so the walk is over those. +fn mutable_variables(expr: &Expr) -> Vec> { + fn go<'e>( + e: &'e Expr, + counts: &mut HashMap, + out: &mut Vec>, + ) { + if let TypedExprNode::Transact { keys, .. } = &e.node { + for k in keys { + let name = k.name.field_key(); + let index = counts.entry(name.clone()).or_insert(0); + out.push(MutableVariable { + path: VarPath { + name, + index: *index, + }, + key: k, + declared_by: e.node_id(), + }); + *index += 1; + } + } + e.walk_children(|c| go(c, counts, out)); + } + let mut counts = HashMap::new(); + let mut out = Vec::new(); + go(expr, &mut counts, &mut out); + out +} + +/// A variable the running program is holding that a new version cannot take +/// over. +/// +/// Both shapes are about the *value* having nowhere to go. A variable whose +/// recurrence now reads something else is not one of them: the value seeds and +/// the position restarts, since a position only means something in the domain it +/// was counted in. +#[derive(Debug)] +pub enum StateConflict { + /// The new version does not declare it, so its value has nowhere to be + /// seeded and would be discarded. + Dropped { path: VarPath }, + /// The new version declares it at a different type. Its value cannot be the + /// seed of a store that expects another shape: the store would be built + /// around a constant of the wrong extent and fail on its first pull. + Retyped { + path: VarPath, + held: Extent, + declared: Extent, + }, +} + +impl StateConflict { + /// The variable this is about, for naming it in a diagnostic. + pub fn path(&self) -> &VarPath { + match self { + StateConflict::Dropped { path } | StateConflict::Retyped { path, .. } => path, + } + } +} + /// Build an induction-domain store (a `mut` loop). Every induction store — plain, /// conditional, or feed-carrying, over a finite or async extent — is single-writer /// (recognition folds a conditional write to one carry-complete writer), so this @@ -1713,6 +2355,8 @@ fn induction_extent_is_positional(extent: &Extent) -> bool { fn build_induction_store( keys: &[TransactKey], writers: &[WriterSite], + domain: &Type, + paths: &[VarPath], ctx: &mut OpConversionContext, ) -> Result { let n_accs = keys.len(); @@ -1743,7 +2387,7 @@ fn build_induction_store( w.write_keys.len(), "the induction writer writes every accumulator key" ); - build_induction_store_single(keys, w, ctx) + build_induction_store_single(keys, w, domain, paths, ctx) } /// Build a single-writer induction store as a position-driven [`InductionStore`] @@ -1759,10 +2403,13 @@ fn build_induction_store( fn build_induction_store_single( keys: &[TransactKey], w: &WriterSite, + domain: &Type, + paths: &[VarPath], ctx: &mut OpConversionContext, ) -> Result { let key_extent = Extent::Base(BaseType::String); let runtime_key = |n: &Name| Value::String(n.field_key().into()); + let domain = strip_refinements(domain); // Each accumulator becomes a mutable variable key: its init op (the fold default, read // once at subscribe) plus a dense-read entry carrying the init as the @@ -1770,27 +2417,59 @@ fn build_induction_store_single( let mut keys_map: HashMap = HashMap::with_capacity(keys.len()); let mut init_ops: Vec<(Value, Box)> = Vec::new(); let mut value_extents: Vec = Vec::new(); - for k in keys { + // Where this store starts, once a carried variable names the frontier of a + // predecessor counting in this same sequence. `None` for a store with no such + // predecessor, which starts where its source does instead. + let mut resume_at: Option = None; + let sequence = sequence_of(&domain, &w.source, ctx); + for (i, k) in keys.iter().enumerate() { let field = k.name.field_key(); let rk = Value::String(field.clone().into()); let value_extent = ctx.extent_of(&k.init.ty)?; if !value_extents.contains(&value_extent) { value_extents.push(value_extent.clone()); } - init_ops.push((rk.clone(), convert_impl(&k.init, None, ctx)?)); + // A variable the replaced version was carrying resumes from the value it + // held; one this version introduces starts from the init it declares. + // Rebuilding a store therefore changes what the loop does next without + // discarding what it had accumulated, which is what distinguishes + // swapping the logic from recomputing the program. + let carried = ctx.inherited.mutable_state.get(&paths[i]); + let resumed = carried + .and_then(|c| c.resumption.as_ref()) + .filter(|r| r.sequence == sequence); + if let Some(resumed) = resumed { + // Every variable of one store carries that store's frontier, so any + // of them answers for the store. They must agree: the seed tick and + // the drive's window base both come from this, and the drive asserts + // that a decision cannot precede the input it decides. + let claimed = *resume_at.get_or_insert(resumed.position); + debug_assert_eq!( + claimed, resumed.position, + "`{field}` resumes at {} where its store's other variables resume at {claimed}", + resumed.position, + ); + } + let init_op: Box = match carried { + Some(carried) => { + trace!("resuming {field} from the retired version's value"); + Box::new(Constant::new(carried.value.clone(), value_extent.clone())) + } + None => convert_impl(&k.init, None, ctx)?, + }; + init_ops.push((rk.clone(), init_op)); keys_map.insert( field, KeyReadInfo { + // An accumulator persists across positions. A literal init is the + // leading-carry fold default; a *conditional* single-writer loop + // does have leading carries (positions before the first + // committing write), and those read the accumulator's seed, + // supplied by the tick-0 init in `CommitEngine::new(inits)` — not + // this default, which anchors a computed-init empty fold. + carried: Some(paths[i].clone()), runtime_key: rk, value_extent, - index: 0, // unused: dense reads fold by runtime_key - carry_forward: true, // an accumulator persists across positions - // A literal init is the leading-carry fold default. A - // *conditional* single-writer loop does have leading carries - // (positions before the first committing write); those read the - // accumulator's seed, supplied by the tick-0 init in - // `CommitEngine::new(inits)` — not this default, which anchors a - // computed-init empty fold. }, ); } @@ -1811,7 +2490,7 @@ fn build_induction_store_single( w.source.ty )) })?; - let induction_extent = ctx.extent_of(&crate::ccl::ccl_utils::strip_refinements(&raw_domain))?; + let induction_extent = ctx.extent_of(&strip_refinements(&raw_domain))?; // The recurrence is sequenced by `UInt` position end to end: the driver pairs // items with `UInt` domain keys, `CommitEngine` ticks are positions, and // `StoreDenseRead` folds tick `p + 1` at each. A product domain (a @@ -1848,8 +2527,8 @@ fn build_induction_store_single( // the same shape a commit writer carries (see `build_commit_store`). Each tap // becomes a write-only changelog key (appended after the accumulator keys), so // its per-position value rides the committing change and is read back densely. - // A tap is a per-position event, not a carried mutable variable (`carry_forward: - // false`): it appears only at the position that fired it. Under a conditional + // A tap is a per-position event, not a carried mutable variable (`carried: + // None`): it appears only at the position that fired it. Under a conditional // feed the decision also carries a `to___fire` gate, which the producer // reads to omit a non-fired tap from the delta. let mut write_keys: Vec = w.write_keys.iter().map(runtime_key).collect(); @@ -1860,10 +2539,9 @@ fn build_induction_store_single( keys_map.insert( field.clone(), KeyReadInfo { + carried: None, // a tap fires only at its own position runtime_key: Value::String(field.clone().into()), value_extent: tap_value_extent, - index: 0, // unused: dense reads fold by runtime_key - carry_forward: false, // a tap fires only at its own position }, ); tap_fields.push(field); @@ -1875,7 +2553,33 @@ fn build_induction_store_single( _ => Extent::Union(TagMap::from_positional(value_extents)), }; - let store = InductionStore::new(init_ops, write_keys, tap_fields, key_extent, value_extent); + // A store replacing one over the same domain resumes at that store's + // frontier, which is deliberately behind its source: a drive reads one + // position back through its input, so the source still owes it an element the + // recurrence has already decided. + // + // Every other store starts where its source will next offer a producer. That + // is `0` for a source nothing has read — every source of a program's first + // version — and the released frontier for one a retired version advanced, + // which is the case for a variable that moved to another loop, a program that + // moved to another port, and a stateful loop a version adds over a source it + // was already reading. Starting such a store at `0` would base its drive below + // every position the source will offer, and it would wait for an element that + // is not coming. + // + // Both the store's seed tick and the drive's window base come from this. + let resume_at = match resume_at { + Some(frontier) => frontier, + None => source_start(&domain, ctx), + }; + let store = InductionStore::new( + init_ops, + write_keys, + tap_fields, + key_extent, + value_extent, + resume_at, + ); let set_body = store.body_input_setter(); // Cyclic: the driver reads this store's changelog back to recover each // position's previous accumulator, so one fan branch feeds the cycle and the @@ -1887,6 +2591,7 @@ fn build_induction_store_single( w.read_keys.iter().map(runtime_key).collect(), read_extents, item_extent, + resume_at, ); set_body(convert_impl(&w.body, Some(Box::new(driver)), ctx)?); Ok(StoreReadInfo { @@ -1894,6 +2599,9 @@ fn build_induction_store_single( keys: keys_map, kind: StoreReadKind::InductionChangelog, induction_extent: Some(induction_extent), + sequence: Some(sequence), + // Set by `bind_store`, which is what knows the store's identity. + correspondent: 0, }) } @@ -2040,8 +2748,7 @@ fn convert_store_read( ( k.runtime_key.clone(), k.value_extent.clone(), - k.index, - k.carry_forward, + k.carry_forward(), ) }); ( @@ -2061,14 +2768,9 @@ fn convert_store_read( // does — which is what makes `await_final(x)` independent of a store-mate // still committing. `final_or_default(stream, init)` then reduces it with // `ExtractFinal`, supplying the seed when the key was never written. - (StoreReadKind::Commit, Some((runtime_key, value_extent, _, carry_forward))) => { - Ok(Box::new(StoreValueStream::new( - fan.branch(), - runtime_key, - value_extent, - carry_forward, - ))) - } + (StoreReadKind::Commit, Some((runtime_key, value_extent, carry_forward))) => Ok(Box::new( + StoreValueStream::new(fan.branch(), runtime_key, value_extent, carry_forward), + )), // An `InductionChangelog` key read off the changelog, folded at every // position of the loop extent via [`StoreDenseRead`] (an `IterateExtent(D)` // trigger + the store branch). An **accumulator** (`carry_forward: true`) @@ -2079,10 +2781,7 @@ fn convert_store_read( // per-position value stream: only the positions where the tap fired // (its value present in that position's changelog delta), keyed by loop // position — the same `Fun(D, V)` the sink reads. - ( - StoreReadKind::InductionChangelog, - Some((runtime_key, value_extent, _, carry_forward)), - ) => { + (StoreReadKind::InductionChangelog, Some((runtime_key, value_extent, carry_forward))) => { let extent = induction_extent.ok_or_else(|| { ConversionError::Unsupported(format!( "induction-changelog store {store_name} has no loop extent" diff --git a/src/interpreter/producer_releases.rs b/src/interpreter/producer_releases.rs new file mode 100644 index 000000000..11472414e --- /dev/null +++ b/src/interpreter/producer_releases.rs @@ -0,0 +1,140 @@ +//! Per-producer release bookkeeping for a data source. + +use std::collections::HashMap; + +use crate::interpreter::tiling::Predicate; + +/// What each producer reading a data source has released, and what a producer +/// registering from now on starts having released. +/// +/// A source retains a value until every producer reading it is finished with it, +/// so the intersection of these — the agreement — is what it may drop. Every +/// source keeps this, and keeps it the same way; naming it once is what makes +/// carrying the agreement across a version handover a property of a source rather +/// than of whichever ones happen to store their releases alike. +#[derive(Debug)] +pub(crate) struct ProducerReleases { + /// What each producer has released, accumulated by union across its releases. + per_producer: HashMap, + /// What a producer registering from now on is recorded as having already + /// released. + /// + /// `False` until [`carry_to_new_producers`](Self::carry_to_new_producers) + /// sets it, so a program's own producers — which all register before it + /// starts consuming — read everything the source holds, including whatever + /// arrived while the program was being compiled. + on_registration: Predicate, +} + +impl Default for ProducerReleases { + fn default() -> Self { + Self { + per_producer: HashMap::new(), + on_registration: Predicate::False, + } + } +} + +impl ProducerReleases { + /// Accumulate `obsolete` into `producer`'s record, registering it at + /// [`on_registration`](Self::on_registration) if this is its first release. + pub(crate) fn record(&mut self, producer: &str, obsolete: &Predicate) { + let recorded = self + .per_producer + .entry(producer.to_string()) + .or_insert_with(|| self.on_registration.clone()); + *recorded = recorded.union(obsolete); + } + + /// What `producer` has released, or `None` for one that has never released. + pub(crate) fn of(&self, producer: &str) -> Option<&Predicate> { + self.per_producer.get(producer) + } + + /// What every registered producer has released — what the source may drop, + /// and what a producer registering from now on may skip. + /// + /// Nobody registered means nobody has released anything. The fold's identity + /// is the universal predicate, which for an empty producer list would + /// otherwise read as "everything". + pub(crate) fn agreed(&self) -> Predicate { + if self.per_producer.is_empty() { + return Predicate::False; + } + self.per_producer + .values() + .fold(Predicate::True, |agreed, released| { + agreed.intersect(released) + }) + } + + /// Record the agreement as the starting point for producers registering from + /// now on. + /// + /// Called when a running program is replaced + /// ([`LiveProgram::update`](crate::live_program::LiveProgram::update)). The + /// operators the replacement rebuilds register as new producers, and a source + /// hands a newly-registered one everything it has retained, so without this + /// the replacement recomputes the program's history instead of continuing it + /// and re-emits an output for every input the replaced version answered. + /// + /// The agreement is the safe answer: an index some producer has not finished + /// with is not skipped, so an element that arrived but went unhandled is still + /// delivered to whoever takes over. + pub(crate) fn carry_to_new_producers(&mut self) { + self.on_registration = self.agreed(); + } + + /// What a producer registering now is recorded as having already released. + pub(crate) fn on_registration(&self) -> &Predicate { + &self.on_registration + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::interpreter::Value; + + /// Nobody registered has released nothing, not everything — the fold's + /// identity is the universal predicate, so an empty list has to be answered + /// before the fold rather than by it. + #[test] + fn an_unread_source_has_released_nothing() { + assert_eq!(ProducerReleases::default().agreed(), Predicate::False); + } + + /// A producer's releases accumulate, and the agreement is the part every + /// producer is finished with — so one producer lagging holds the whole + /// agreement at where it has got to. + #[test] + fn the_agreement_is_where_the_slowest_producer_has_reached() { + let mut releases = ProducerReleases::default(); + releases.record("a", &Predicate::LessThanEq(Value::UInt(1))); + releases.record("a", &Predicate::LessThanEq(Value::UInt(4))); + releases.record("b", &Predicate::LessThanEq(Value::UInt(2))); + assert_eq!(releases.agreed(), Predicate::LessThanEq(Value::UInt(2))); + } + + /// A producer registering before the carry reads everything the source holds; + /// one registering after starts at the agreement, which is what makes a + /// rebuilt operator continue the stream rather than reprocess it. + #[test] + fn a_producer_registering_after_the_carry_starts_at_the_agreement() { + let mut releases = ProducerReleases::default(); + releases.record("a", &Predicate::LessThanEq(Value::UInt(2))); + assert_eq!(releases.on_registration(), &Predicate::False); + + releases.carry_to_new_producers(); + assert_eq!( + releases.on_registration(), + &Predicate::LessThanEq(Value::UInt(2)) + ); + + // The newcomer counts as having released the agreement even before it + // releases anything itself, so it neither re-reads what is gone nor holds + // the agreement back to nothing. + releases.record("b", &Predicate::False); + assert_eq!(releases.agreed(), Predicate::LessThanEq(Value::UInt(2))); + } +} diff --git a/src/interpreter/scheduler.rs b/src/interpreter/scheduler.rs index 12218b33e..490e3b4be 100644 --- a/src/interpreter/scheduler.rs +++ b/src/interpreter/scheduler.rs @@ -1,4 +1,8 @@ -use std::{cell::RefCell, collections::HashMap, rc::Rc}; +use std::{ + cell::RefCell, + collections::HashMap, + rc::{Rc, Weak}, +}; use crate::interpreter::{Consumer, DataSourceDomainExtentImpl}; @@ -75,7 +79,7 @@ pub struct Scheduler { type SourceHandle = ( Rc>, - Vec>, + Vec>>, ); impl Scheduler { @@ -83,10 +87,19 @@ impl Scheduler { Self::default() } + /// Register `consumer` to be notified when `handle` has new data. + /// + /// The registration is **weak**, and the subscriber that made it owns the + /// consumer. A registration lasts exactly as long as the producer it wakes, + /// which is what a source handle outliving the graph subscribed to it + /// requires: replacing a program drops the operators it rebuilt, pruning + /// their registrations, while an operator carried across the replacement + /// keeps waking as before. A strong registration would instead keep every + /// operator any version ever subscribed alive and being notified. pub fn add_source_handle( &mut self, handle: Rc>, - consumer: Box, + consumer: Weak>, ) { let id = handle.borrow().get_id().to_string(); if let Some(entry) = self.source_handles.get_mut(&id) { @@ -108,8 +121,13 @@ impl Scheduler { self.source_handles .values_mut() .for_each(|(source, consumers)| { + // Prune first, so a source whose every subscriber is gone stops + // accumulating dead registrations across program updates. + consumers.retain(|c| c.strong_count() > 0); if source.borrow_mut().check_for_new_data() { - consumers.iter_mut().for_each(|c| c.notify()); + for consumer in consumers.iter().filter_map(Weak::upgrade) { + consumer.borrow_mut().notify(); + } } }); // Deliver deferred wakeups now — outside any `get`, so a notification diff --git a/src/interpreter/sinks.rs b/src/interpreter/sinks.rs index 2e9b862f7..a9606b881 100644 --- a/src/interpreter/sinks.rs +++ b/src/interpreter/sinks.rs @@ -72,6 +72,12 @@ impl DoneNotifier { /// filled after [`crate::interpreter::tile_operators::TileOperator::subscribe`] returns (solving the chicken-and-egg: /// the consumer must exist before subscribe is called, but subscribe is what /// creates the producer). +/// +/// So a notification raised *during* `subscribe` — an induction store raises one +/// to start its loop — arrives with the slot still empty and is dropped. Whoever +/// fills the slot notifies once afterwards for that reason +/// ([`crate::ccl::context::compile_program`]); a version installed while work was +/// outstanding is otherwise never pulled. pub struct SinkConsumer { /// The compiled responses producer, filled in after subscribe returns. producer: ProducerSlot, @@ -84,9 +90,11 @@ pub struct SinkConsumer { impl SinkConsumer { /// Create a new consumer paired with `sink` and `done`. /// - /// The returned consumer holds a shared handle to the `producer` slot; the - /// caller should fill that slot with the `TileProducer` returned by - /// [`crate::interpreter::tile_operators::TileOperator::subscribe`] before the first notification fires. + /// The returned consumer holds a shared handle to the `producer` slot. The + /// caller fills it with the `TileProducer` returned by + /// [`crate::interpreter::tile_operators::TileOperator::subscribe`] and then + /// notifies once, which is the only way this consumer sees the notifications + /// `subscribe` itself raised. pub fn new(sink: Arc, done: DoneNotifier) -> (Self, ProducerSlot) { let slot: ProducerSlot = Rc::new(RefCell::new(None)); ( @@ -99,6 +107,20 @@ impl SinkConsumer { ) } + /// Stop dispatching and release the producer chain behind this consumer. + /// + /// A [`DataSink`] outlives any one version of a program; the subscription + /// that feeds it does not. Detaching is what ends a replaced version's + /// dispatch, and it is not achieved by dropping the consumer alone: an + /// operator carried across the replacement still holds the notification + /// closure that reaches this consumer, so the replaced version would keep + /// being woken and keep writing to a sink its successor now owns. Clearing + /// the producer slot also drops the operators behind it, which is what lets + /// the fan-outs they subscribed to see those subscriptions end. + pub fn detach(&mut self) { + *self.producer.borrow_mut() = None; + } + /// Call `f` with the sink's current producer, if it has been set. pub fn with_producer(&self, f: F) { if let Some(ref prod) = *self.producer.borrow() { diff --git a/src/interpreter/stdio.rs b/src/interpreter/stdio.rs index 674a55687..8a9890331 100644 --- a/src/interpreter/stdio.rs +++ b/src/interpreter/stdio.rs @@ -128,6 +128,14 @@ impl DataSourceDomainExtentImpl for StdinDataSource { fn release(&mut self, producer: &str, obsolete: Predicate) { self.buf.release(producer, obsolete); } + + fn carry_release_to_new_producers(&mut self) { + self.buf.releases.carry_to_new_producers(); + } + + fn first_position_for_a_new_producer(&self) -> usize { + self.buf.first_index_for_a_new_producer() + } } #[cfg(test)] @@ -144,10 +152,7 @@ mod tests { source.add("a".into()); source.add("b".into()); source.add("c".into()); - source - .buf - .obsolete_predicates - .insert("p".to_string(), Predicate::False); + source.buf.releases.record("p", &Predicate::False); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![0, 1, 2])); @@ -164,8 +169,8 @@ mod tests { source.add("d".into()); source .buf - .obsolete_predicates - .insert("p".to_string(), Predicate::LessThanEq(Value::UInt(1))); + .releases + .record("p", &Predicate::LessThanEq(Value::UInt(1))); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![2, 3])); @@ -177,10 +182,7 @@ mod tests { let mut source = StdinDataSource::new(); source.add("a".into()); source.add("b".into()); - source - .buf - .obsolete_predicates - .insert("p".to_string(), Predicate::True); + source.buf.releases.record("p", &Predicate::True); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![])); @@ -196,10 +198,7 @@ mod tests { } // Mark indices 1 and 2 as obsolete; live window [0,4] minus {1,2} = {0,3,4}. let filter = Predicate::from_column_value(&ColumnValue::UInts(vec![1, 2])); - source - .buf - .obsolete_predicates - .insert("p".to_string(), filter); + source.buf.releases.record("p", &filter); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![0, 3, 4])); @@ -218,10 +217,7 @@ mod tests { // Mark index 3 as obsolete; live window [2,4] minus {3} = {2,4}. let filter = Predicate::from_column_value(&ColumnValue::UInts(vec![3])); - source - .buf - .obsolete_predicates - .insert("p".to_string(), filter); + source.buf.releases.record("p", &filter); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![2, 4])); @@ -234,10 +230,7 @@ mod tests { let mut source = StdinDataSource::new(); // No lines added; start_idx == ready_size == 0. let filter = Predicate::from_column_value(&ColumnValue::UInts(vec![0])); - source - .buf - .obsolete_predicates - .insert("p".to_string(), filter); + source.buf.releases.record("p", &filter); let result = source.get_elements("p"); assert_eq!(result, ColumnValue::from_uints(vec![])); @@ -312,7 +305,7 @@ mod tests { // Verify producer_a's predicate is recorded assert!( - source.buf.obsolete_predicates.contains_key("producer_a"), + source.buf.releases.of("producer_a").is_some(), "producer_a predicate should be stored" ); @@ -322,11 +315,11 @@ mod tests { // Verify both predicates are recorded assert!( - source.buf.obsolete_predicates.contains_key("producer_a"), + source.buf.releases.of("producer_a").is_some(), "producer_a predicate should still be stored" ); assert!( - source.buf.obsolete_predicates.contains_key("producer_b"), + source.buf.releases.of("producer_b").is_some(), "producer_b predicate should be stored" ); @@ -352,24 +345,14 @@ mod tests { source.release("producer_a", Predicate::LessThanEq(Value::UInt(0))); // Store the first predicate - let pred_after_first = source - .buf - .obsolete_predicates - .get("producer_a") - .cloned() - .unwrap(); + let pred_after_first = source.buf.releases.of("producer_a").cloned().unwrap(); // Second release from producer A: index 1 // This should use union with the existing predicate source.release("producer_a", Predicate::LessThanEq(Value::UInt(1))); // The predicate should now be an OR of the two - let pred_after_second = source - .buf - .obsolete_predicates - .get("producer_a") - .cloned() - .unwrap(); + let pred_after_second = source.buf.releases.of("producer_a").cloned().unwrap(); // The second predicate should be different from the first (should be OR'd) assert_ne!( diff --git a/src/interpreter/stream_buffer.rs b/src/interpreter/stream_buffer.rs index c40fb58b8..8f7d6d9a1 100644 --- a/src/interpreter/stream_buffer.rs +++ b/src/interpreter/stream_buffer.rs @@ -1,10 +1,10 @@ //! Shared buffer for uint-indexed string stream sources. //! -//! [`UIntStreamBuffer`] captures the buffer, sliding-window indexing, and -//! per-producer obsolete-predicate bookkeeping that is common to every -//! streaming `UInt → String` data source (stdin, HTTP server, etc.). - -use std::collections::HashMap; +//! [`UIntStreamBuffer`] captures the buffer and the sliding-window indexing +//! common to every streaming `UInt → String` data source (stdin, HTTP server, +//! etc.). Which producer has released what is +//! [`ProducerReleases`](crate::interpreter::producer_releases::ProducerReleases), +//! which every source keeps rather than only these. use intervalsets::{ Bounding, Interval, IntervalSet, @@ -13,7 +13,9 @@ use intervalsets::{ use log::trace; use smol_str::SmolStr; -use crate::interpreter::{ColumnValue, Value, tiling::Predicate}; +use crate::interpreter::{ + ColumnValue, Value, producer_releases::ProducerReleases, tiling::Predicate, +}; /// Buffer and predicate bookkeeping for a uint-indexed string stream. /// @@ -37,8 +39,9 @@ pub(crate) struct UIntStreamBuffer { /// `true` once a universal release has been received. closed: bool, - /// Per-producer obsolete predicates, accumulated via union on each [`release`]. - pub(crate) obsolete_predicates: HashMap, + /// What each producer has released, and where one registering from now on + /// starts. + pub(crate) releases: ProducerReleases, } impl UIntStreamBuffer { @@ -49,7 +52,7 @@ impl UIntStreamBuffer { ready_size: 0, eof_reached: false, closed: false, - obsolete_predicates: HashMap::new(), + releases: ProducerReleases::default(), } } @@ -160,8 +163,8 @@ impl UIntStreamBuffer { /// Return the non-obsolete indices in `[start_idx, ready_size)` for `producer`. pub(crate) fn get_elements(&self, producer: &str) -> ColumnValue { let obsolete = self - .obsolete_predicates - .get(producer) + .releases + .of(producer) .unwrap_or_else(|| panic!("Unknown producer: {producer}")); let live = self.live_window().difference(&Self::index_set(obsolete)); let mut indices = Vec::new(); @@ -176,35 +179,37 @@ impl UIntStreamBuffer { ColumnValue::from_uints(indices) } + /// The first index a producer registering now will be offered. + /// + /// The buffer has already dropped everything every producer agreed on, so + /// this is `start_idx` unless the carried release runs past it. A store built + /// over this source starts here: the source will never offer the positions + /// below it, and a drive based lower waits for an element that is not coming. + pub(crate) fn first_index_for_a_new_producer(&self) -> usize { + match self.released_prefix(&Self::index_set(self.releases.on_registration())) { + Some(last) => last + 1, + None => self.start_idx, + } + } + /// Update per-producer obsolete predicates and release any buffer entries /// that all producers agree are no longer needed. pub(crate) fn release(&mut self, producer: &str, obsolete: Predicate) { - let recorded = self - .obsolete_predicates - .entry(producer.to_string()) - .or_insert(Predicate::False); - *recorded = recorded.union(&obsolete); + self.releases.record(producer, &obsolete); // Every registered producer is done with every index, so nothing will be // read again. A producer with no entry has not subscribed, and so holds // nothing back. - if self.obsolete_predicates.values().all(Predicate::is_true) { + let agreed = self.releases.agreed(); + if agreed.is_true() { self.close(); return; } - // Each producer's accumulated set is every index it will not read again, - // so their intersection is what the buffer may drop. The intersection is - // over those accumulated sets rather than over this release and them: - // a producer's earlier releases still stand. Every recorded guard passes - // through `index_set` here, which is where one built for another extent - // fails. - let agreed = self - .obsolete_predicates - .values() - .fold(IntervalSet::from(Interval::unbounded()), |agreed, pred| { - agreed.intersection(&Self::index_set(pred)) - }); + // The agreement is every index no producer will read again, so it is what + // the buffer may drop. A guard built for another extent is rejected the + // first time one is read as an index set — here, or in `get_elements`. + let agreed = Self::index_set(&agreed); trace!("UIntStreamBuffer::release: {agreed:?}"); if let Some(i) = self.released_prefix(&agreed) { self.release_index(i); @@ -241,6 +246,8 @@ impl UIntStreamBuffer { #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; fn buffer_with(n: usize) -> UIntStreamBuffer { @@ -291,7 +298,7 @@ mod tests { /// is the intersection across producers, not the latest one to arrive. /// /// Both producers register before any release, which is what subscribing - /// does (`IterateExtent::subscribe` releases `Predicate::False` to enrol + /// does (`IterateExtent::subscribe` releases `Predicate::False` to register /// with each source in its extent). A producer with no entry is not in the /// intersection and so does not hold anything back. #[test] @@ -384,4 +391,119 @@ mod tests { let mut buf = buffer_with(8); buf.release("p", Predicate::LessThanEq(Value::Int(4))); } + + /// A producer registering before any release has been carried reads the + /// whole buffer, including values that arrived before it registered. + /// + /// This is a program's own producers, which all enrol during compilation: + /// anything that arrived while the program was being compiled is still + /// theirs to read. + #[test] + fn a_producer_registering_at_the_start_reads_everything() { + let mut buf = buffer_with(3); + buf.release("p", Predicate::False); + assert_eq!( + buf.get_elements("p"), + ColumnValue::from_uints(vec![0, 1, 2]) + ); + } + + /// Once the release state is carried, a producer registering from then on + /// reads only what arrives next. + /// + /// This is what stops a replacement version reprocessing the stream: the + /// operators it rebuilds enrol as new producers, and a source hands a + /// A producer registering once everything buffered has been released reads + /// only what arrives next. + #[test] + fn a_producer_registering_after_the_carry_reads_only_what_follows() { + let mut buf = buffer_with(3); + buf.release("p", Predicate::False); + buf.release("p", covering(0, 2)); + + buf.releases.carry_to_new_producers(); + buf.release("late", Predicate::False); + assert_eq!( + buf.get_elements("late"), + ColumnValue::from_uints(Vec::new()), + "`p` handled every buffered index, so none is the new producer's" + ); + + buf.push(SmolStr::new("e3")); + assert_eq!( + buf.get_elements("late"), + ColumnValue::from_uints(vec![3]), + "and what arrives next is" + ); + } + + /// Carrying the release state does not retroactively change a producer that + /// had already registered. + #[test] + fn carrying_the_release_state_leaves_registered_producers_alone() { + let mut buf = buffer_with(3); + buf.release("early", Predicate::False); + buf.releases.carry_to_new_producers(); + assert_eq!( + buf.get_elements("early"), + ColumnValue::from_uints(vec![0, 1, 2]), + "it applies at registration, not to whoever is already reading" + ); + } + + /// A producer that registers late still holds the buffer: the prefix is only + /// freed once it has released it too. + #[test] + fn a_late_producer_still_counts_toward_the_release_intersection() { + let mut buf = buffer_with(3); + buf.release("early", Predicate::False); + buf.release("early", covering(0, 2)); + buf.releases.carry_to_new_producers(); + buf.release("late", Predicate::False); + + assert_eq!( + buf.start_idx, 3, + "both producers have released 0..=2, so the prefix frees" + ); + } + + /// What a late producer skips is what every producer has *released*, not + /// everything that has arrived. + /// + /// The regression this pins: taking the end of the buffer instead meant an + /// index that had arrived but that nobody had finished was skipped by the + /// producer taking over, so it was never handled by anyone. For an HTTP + /// source that is a request accepted and then silently never answered. + #[test] + fn a_late_producer_inherits_the_release_state_not_the_buffer_end() { + let mut buf = buffer_with(3); + buf.release("p", Predicate::False); + // `p` finished index 0; 1 and 2 have arrived and nobody has handled them. + buf.release("p", covering(0, 0)); + + buf.releases.carry_to_new_producers(); + buf.release("late", Predicate::False); + assert_eq!( + buf.get_elements("late"), + ColumnValue::from_uints(vec![1, 2]), + "the unfinished indices are still the new producer's to read" + ); + } + + /// A release that is not a prefix lets a new producer skip only the released + /// indices, not the gap below them. + #[test] + fn a_non_prefix_release_starts_a_new_producer_at_the_gap() { + let mut buf = buffer_with(5); + buf.release("p", Predicate::False); + buf.release("p", covering(2, 4)); + + buf.releases.carry_to_new_producers(); + buf.release("late", Predicate::False); + assert_eq!( + buf.get_elements("late"), + ColumnValue::from_uints(vec![0, 1]), + "and the new producer reads exactly what is unreleased" + ); + } } diff --git a/src/interpreter/test_source.rs b/src/interpreter/test_source.rs index 0bd800b09..4e0b89910 100644 --- a/src/interpreter/test_source.rs +++ b/src/interpreter/test_source.rs @@ -5,7 +5,10 @@ use log::trace; use crate::{ ccl::Type, - interpreter::{ColumnValue, DataSourceDomainExtentImpl, Extent, Value, tiling::Predicate}, + interpreter::{ + ColumnValue, DataSourceDomainExtentImpl, Extent, Value, + producer_releases::ProducerReleases, tiling::Predicate, + }, }; /// Handle for simulating an arbitrary source in a program. @@ -21,7 +24,7 @@ pub struct TestDataSource { yield_predicate: Predicate, has_data: bool, data: HashMap, - obsolete_predicates: HashMap, + releases: ProducerReleases, } impl TestDataSource { @@ -35,7 +38,7 @@ impl TestDataSource { yield_predicate: Predicate::False, has_data: false, data: HashMap::new(), - obsolete_predicates: HashMap::new(), + releases: ProducerReleases::default(), } } @@ -64,11 +67,7 @@ impl TestDataSource { /// Returns a predicate corresponding to the data that has been entirely released from the source. pub fn get_released_predicate(&self) -> Predicate { - let mut result = Predicate::True; - for pred in self.obsolete_predicates.values() { - result = result.intersect(pred); - } - result + self.releases.agreed() } } @@ -85,8 +84,8 @@ impl DataSourceDomainExtentImpl for TestDataSource { fn get_elements(&self, producer: &str) -> ColumnValue { let filter = self - .obsolete_predicates - .get(producer) + .releases + .of(producer) .unwrap_or_else(|| panic!("Unknown producer: {}", producer)); trace!("Iterating test source elements with filter {filter:?}"); ColumnValue::from_values( @@ -129,22 +128,27 @@ impl DataSourceDomainExtentImpl for TestDataSource { self.yield_predicate.clone() } - fn release(&mut self, producer: &str, mut obsolete: Predicate) { + fn release(&mut self, producer: &str, obsolete: Predicate) { trace!( - "TestDataSource::release: {obsolete:?} with obsolete predicates: {:?}", - self.obsolete_predicates + "TestDataSource::release: {obsolete:?} with {:?}", + self.releases ); - let pred = self - .obsolete_predicates - .entry(producer.to_string()) - .or_insert(Predicate::False); - *pred = pred.union(&obsolete); - for pred in self.obsolete_predicates.values() { - obsolete = obsolete.intersect(pred); - } - trace!("TestDataSource::release: intersected to {obsolete:?}"); - self.data - .retain(|k, _| !key_matches_predicate(k, &obsolete)); + self.releases.record(producer, &obsolete); + let agreed = self.releases.agreed(); + trace!("TestDataSource::release: intersected to {agreed:?}"); + self.data.retain(|k, _| !key_matches_predicate(k, &agreed)); + } + + fn carry_release_to_new_producers(&mut self) { + self.releases.carry_to_new_producers(); + } + + fn first_position_for_a_new_producer(&self) -> usize { + // A test source's keys are arbitrary values rather than stream positions, + // so it names no position for a store to start at. A store over one starts + // at `0` and reads whatever the source still offers a new producer, which + // the carry above bounds. + 0 } } @@ -239,7 +243,7 @@ mod tests { // Verify producer_a's predicate is stored assert!( - source.obsolete_predicates.contains_key("producer_a"), + source.releases.of("producer_a").is_some(), "producer_a predicate should be recorded" ); @@ -248,11 +252,11 @@ mod tests { // Verify both predicates are stored assert!( - source.obsolete_predicates.contains_key("producer_a"), + source.releases.of("producer_a").is_some(), "producer_a should still be recorded" ); assert!( - source.obsolete_predicates.contains_key("producer_b"), + source.releases.of("producer_b").is_some(), "producer_b should be recorded" ); @@ -281,9 +285,9 @@ mod tests { // First release from producer A source.release("producer_a", Predicate::LessThanEq(Value::UInt(0))); - // Verify producer_a is in the obsolete_predicates + // Verify producer_a has a record assert!( - source.obsolete_predicates.contains_key("producer_a"), + source.releases.of("producer_a").is_some(), "producer_a should be recorded" ); @@ -294,11 +298,11 @@ mod tests { // Verify both are recorded assert!( - source.obsolete_predicates.contains_key("producer_a"), + source.releases.of("producer_a").is_some(), "producer_a should still be recorded" ); assert!( - source.obsolete_predicates.contains_key("producer_b"), + source.releases.of("producer_b").is_some(), "producer_b should be recorded" ); diff --git a/src/interpreter/tile_operators/fanout.rs b/src/interpreter/tile_operators/fanout.rs index bf7d6dca7..71ed56cb3 100644 --- a/src/interpreter/tile_operators/fanout.rs +++ b/src/interpreter/tile_operators/fanout.rs @@ -1,5 +1,8 @@ use log::trace; -use std::{cell::RefCell, rc::Rc}; +use std::{ + cell::{Cell, RefCell}, + rc::{Rc, Weak}, +}; use super::*; use crate::{ @@ -63,12 +66,98 @@ struct FanOutShared { consumers: Vec>>>, /// Per-subscriber release guards; intersected before passing upstream. release_guards: Vec, + /// What every subscriber had agreed to release the last time one released, + /// and so what this fan-out has already passed upstream. + /// + /// A subscriber registering from now on starts here rather than at nothing. + /// The region is gone: the fan-out told its input it would not be read again, + /// so the input is free to have dropped it and a late subscriber that claimed + /// to still want it would hold the intersection back at a frontier no one is + /// waiting on. Within one version every subscription is made before any data + /// flows, so this is the empty guard and seeding from it changes nothing; + /// across a version handover it is what a rebuilt subscriber inherits from the + /// one it replaces. + released: TileGuard, + /// Each subscriber's slot number, parallel to [`release_guards`] and + /// [`consumers`]. + /// + /// The [`FanOutProducer`] a subscription handed out owns the strong side, so + /// a dead entry means that subscriber's producer has been dropped. Its slot + /// is then skipped: it neither blocks the release intersection nor gets + /// notified. + /// + /// A `Cell` rather than a bare token because the slot number *is* the + /// subscription's identity, and [`compact`](Self::compact) renumbers. A + /// producer reads its index out of the cell it shares with this entry, so + /// dropping dead slots stays compatible with addressing a guard by index: + /// the survivors are told their new numbers. Without that the list would + /// grow by one dead slot per replaced subscriber on every update, forever, + /// and both the notify walk and the release intersection scan it. + /// + /// [`release_guards`]: FanOutShared::release_guards + /// [`consumers`]: FanOutShared::consumers + subscribers: Vec>>, /// Re-entrancy bookkeeping for cyclic op graphs. `None` for non-cyclic /// fan-outs (the overwhelming majority); `Some` only when constructed /// via [`FanOut::new_cyclic`]. reentrancy: Option, } +impl FanOutShared { + /// The slots whose subscriber still exists, in subscription order. + fn live_indices(&self) -> impl Iterator + '_ { + self.subscribers + .iter() + .enumerate() + .filter(|(_, s)| s.strong_count() > 0) + .map(|(i, _)| i) + } + + /// Drop every slot whose subscriber is gone, renumbering the survivors. + /// + /// Each surviving producer learns its new index through the `Cell` it shares + /// with its entry in [`subscribers`](Self::subscribers), so the three + /// parallel vectors stay bounded by the number of live subscriptions rather + /// than by the number ever made. + /// + /// Safe only when no producer is mid-pull, since a renumber between a + /// producer reading its index and using it would address another + /// subscriber's guard. [`FanOut::reopen`] is the one caller, and it runs at a + /// version handover with the graph already torn down. + fn compact(&mut self) { + // Upgrade once: `keep` and the renumbering must agree on which slots are + // live, and reading liveness twice would let them disagree. + let live: Vec<_> = self.subscribers.iter().map(Weak::upgrade).collect(); + if live.iter().all(Option::is_some) { + return; + } + for (next, cell) in live.iter().flatten().enumerate() { + cell.set(next); + } + let keep: Vec = live.iter().map(Option::is_some).collect(); + retain_flagged(&mut self.subscribers, &keep); + retain_flagged(&mut self.consumers, &keep); + retain_flagged(&mut self.release_guards, &keep); + debug_assert_eq!( + self.consumers.len(), + self.subscribers.len(), + "consumers stay parallel to subscribers" + ); + debug_assert_eq!( + self.release_guards.len(), + self.subscribers.len(), + "release guards stay parallel to subscribers" + ); + } +} + +/// Keep the elements of `v` whose flag in `keep` is set, in order. +fn retain_flagged(v: &mut Vec, keep: &[bool]) { + debug_assert_eq!(v.len(), keep.len(), "one flag per element"); + let mut flags = keep.iter(); + v.retain(|_| *flags.next().unwrap_or(&false)); +} + /// RAII guard for the cyclic `FanOut` `subscribing_inner` flag. Created /// when we set the flag `true` to drive the inner `subscribe`; its `Drop` /// resets the flag to `false`. This makes the path panic-safe — if @@ -165,6 +254,8 @@ impl FanOut { producer: None, consumers: Vec::new(), release_guards: Vec::new(), + released: tiling.empty_guard(), + subscribers: Vec::new(), reentrancy, })); Self { @@ -192,6 +283,48 @@ impl FanOut { pub fn tiling(&self) -> &Tiling { &self.tiling } + + /// The tile this fan-out most recently served, for a cyclic fan-out; + /// `None` for an ordinary one, which keeps no memo. + /// + /// Reads the cyclic-mode memo rather than pulling the input, so it is safe + /// wherever the graph is not mid-traversal and observes exactly what the + /// fan's consumers last saw. A store's value is carried on its fan, so this + /// is how a version replacing this program reads what its variables hold + /// without a second channel out of the operator. + pub fn cached_tile(&self) -> Option { + self.shared + .borrow() + .reentrancy + .as_ref() + .map(|r| r.cached_tile.clone()) + } + + /// Whether every subscriber has released everything this fan-out could + /// offer. + /// + /// Such a fan-out answers empty from here on: it has told its input that + /// nothing will be read again, and a `Memo` input drops what it holds in + /// response. A version handing its operators on offers only the ones that can + /// still produce, since adopting this one binds a name to nothing. + pub fn released_in_full(&self) -> bool { + self.shared.borrow().released.is_universal() + } + + /// Reopen this fan-out for a fresh set of branches, keeping the inner + /// producer and everything it has accumulated. + /// + /// For carrying one operator across a program update. Only the + /// [`inspect`](TileOperator::inspect) bookkeeping resets: which branch + /// renders the input subtree and which renders a back-reference. The + /// subscriptions need no attention, because each is tied to the life of the + /// producer it handed out ([`FanOutShared::subscribers`]) — a subscriber the + /// update dropped stops counting on its own, and one the update carried + /// forward keeps its guard. + pub fn reopen(&self) { + *self.used.borrow_mut() = false; + self.shared.borrow_mut().compact(); + } } struct FanOutBranch { @@ -229,14 +362,17 @@ impl TileOperator for FanOutBranch { consumer: Box, scheduler: &mut Scheduler, ) -> Box { - // Register the consumer and reserve its release-guard slot. - let index = { + // Register the consumer and reserve its release-guard slot. The slot + // number lives in the cell the producer holds, so `compact` can renumber. + let slot = Rc::new(Cell::new(0usize)); + { let mut shared = self.shared.borrow_mut(); - let index = shared.consumers.len(); + slot.set(shared.consumers.len()); shared.consumers.push(Rc::new(RefCell::new(consumer))); - shared.release_guards.push(self.tiling.empty_guard()); - index - }; // borrow released here before we might call input.subscribe + let carried = shared.released.clone(); + shared.release_guards.push(carried); + shared.subscribers.push(Rc::downgrade(&slot)); + } // borrow released here before we might call input.subscribe // Decide whether *this* call should drive the inner subscribe. // `producer.is_none()` is the standard "first subscription" check; @@ -283,7 +419,13 @@ impl TileOperator for FanOutBranch { // prevents a re-entrant panic when a consumer (e.g. // SinkConsumer) calls FanOutProducer::get_impl(), which // needs shared.borrow_mut() for the same Rc. - let consumers = shared_rc.borrow().consumers.clone(); + let consumers = { + let shared = shared_rc.borrow(); + shared + .live_indices() + .map(|i| shared.consumers[i].clone()) + .collect::>() + }; for c in &consumers { c.borrow_mut().notify(); } @@ -297,7 +439,7 @@ impl TileOperator for FanOutBranch { Box::new(FanOutProducer { base: ProducerBase::new(self.shared.borrow().id, &self.tiling), shared: self.shared.clone(), - index, + slot, }) } @@ -310,15 +452,30 @@ struct FanOutProducer { base: ProducerBase, /// Shared state (consumers + release guards). shared: Rc>, - /// This producer's index into `shared.consumers` and `shared.release_guards`. - index: usize, + /// This producer's index into `shared.consumers` and `shared.release_guards`, + /// and the token that keeps its slot counted for as long as the producer + /// exists — one object, since the index *is* the subscription's identity. + /// Written by [`FanOutShared::compact`]. See [`FanOutShared::subscribers`]. + // shared-state-ok: which slot this subscription owns — bookkeeping between a + // producer and the fan-out it subscribed to, not a back channel for data. No + // tile, tile guard, or program value passes through it; a producer only reads + // it to index its own release guard, which it would have done with a plain + // `usize` if dead slots never had to be reclaimed. + slot: Rc>, +} + +impl FanOutProducer { + /// This producer's current slot number. + fn index(&self) -> usize { + self.slot.get() + } } impl TileProducer for FanOutProducer { impl_producer_base!(); fn inspect(&self, opts: &VizOptions) -> InspectNode { - if self.index == 0 { + if self.index() == 0 { InspectNode::new(self.name()) .with_tiling(self.tiling().to_string()) .child( @@ -396,7 +553,7 @@ impl TileProducer for FanOutProducer { // Filter by the stored obsolete guard. Because upstream retains data according to the // intersection of all obsolete guards, it may have more data than this specific consumer // is interested in. - let guard = self.shared.borrow().release_guards[self.index].clone(); + let guard = self.shared.borrow().release_guards[self.index()].clone(); trace!("{} removing {guard:?} from {result:?}", self.name()); result.remove_guarded(guard); result @@ -408,13 +565,20 @@ impl TileProducer for FanOutProducer { // delivered data grows monotonically. Replacing (instead of union-ing) // would forget previously-released ranges, causing FanOutBranch to // re-deliver data that a consumer has already released. - let accumulated = shared.release_guards[self.index].union(&obsolete_guard); - shared.release_guards[self.index] = accumulated; + let index = self.index(); + let accumulated = shared.release_guards[index].union(&obsolete_guard); + shared.release_guards[index] = accumulated; + // Only live subscribers constrain the release. A subscriber whose + // producer has been dropped never releases again, so counting its guard + // would hold the intersection wherever that subscriber left it and the + // input would retain everything from there on. let intersection = shared - .release_guards - .iter() - .fold(self.tiling().universal_guard(), |acc, g| acc.intersect(g)); + .live_indices() + .fold(self.tiling().universal_guard(), |acc, i| { + acc.intersect(&shared.release_guards[i]) + }); trace!("{} releasing: {intersection:?}", self.name()); + shared.released = intersection.clone(); // In cyclic mode the inner producer can be temporarily taken out // by a sibling-branch `get_impl`; skip the inner release in that // case (the next non-reentrant release will recompute and @@ -548,7 +712,92 @@ impl TileProducer for MemoProducer { mod tests { use super::*; use crate::interpreter::tile_operators::test_helpers::QuietSpy; - use crate::interpreter::{BaseType, ColumnValue, Extent}; + use crate::interpreter::tile_operators::{Constant, Scheduler}; + use crate::interpreter::{BaseType, ColumnValue, Extent, Value}; + + /// Reopening a fan-out drops the slots whose subscribers are gone and tells + /// each survivor its new number, so the parallel slot vectors are bounded by + /// the live subscriptions rather than by every subscription ever made. + /// + /// The renumbering is the whole point: a producer addresses its release guard + /// by index, so a compaction that moved guards without telling the producers + /// would hand one subscriber another's guard. This drops the *first* of two + /// subscribers for that reason — the survivor has to move from slot 1 to slot + /// 0 and keep the guard it released. + #[test] + fn reopening_a_fan_out_drops_dead_slots_and_renumbers_the_rest() { + let extent = Extent::Base(BaseType::Int); + let fan = FanOut::new(Box::new(Constant::new(Value::Int(1), extent.clone()))); + let mut sched = Scheduler::new(); + let tiling = Tiling::Scalar(extent); + + let first = fan + .branch() + .subscribe(tiling.empty_guard(), Box::new(|| {}), &mut sched); + let mut second = fan + .branch() + .subscribe(tiling.empty_guard(), Box::new(|| {}), &mut sched); + assert_eq!(fan.shared.borrow().subscribers.len(), 2); + + // Distinguish the survivor's guard from the empty one the dead slot holds. + let mine = TileGuard::Scalar(true); + second.release(mine.clone()); + drop(first); + + fan.reopen(); + + let shared = fan.shared.borrow(); + assert_eq!(shared.subscribers.len(), 1, "the dead slot is gone"); + assert_eq!(shared.consumers.len(), 1, "consumers stay parallel"); + assert_eq!(shared.release_guards.len(), 1, "guards stay parallel"); + drop(shared); + assert_eq!( + fan.shared.borrow().release_guards[0], + mine, + "the survivor's guard moved with it, and it reads its new slot" + ); + } + + /// A subscriber that registers after the fan-out has released starts having + /// released the same, because that data is gone: the fan-out told its input + /// it would not be read again. + /// + /// Within one version every subscription is made before any data flows, so + /// this only bites across a version handover — which is exactly when it has + /// to, since the subscriber registering is the one replacing the subscriber + /// that released. + #[test] + fn a_late_subscriber_starts_at_what_the_fan_out_has_released() { + let extent = Extent::Base(BaseType::Int); + let fan = FanOut::new(Box::new(Constant::new(Value::Int(1), extent.clone()))); + let mut sched = Scheduler::new(); + let tiling = Tiling::Scalar(extent); + + let mut first = fan + .branch() + .subscribe(tiling.empty_guard(), Box::new(|| {}), &mut sched); + assert!( + !fan.released_in_full(), + "nothing has released, so the fan-out still has its value to give" + ); + first.release(TileGuard::Scalar(true)); + drop(first); + fan.reopen(); + + assert!( + fan.released_in_full(), + "the one subscriber released everything, so the fan-out can only answer empty" + ); + let late = fan + .branch() + .subscribe(tiling.empty_guard(), Box::new(|| {}), &mut sched); + assert_eq!( + fan.shared.borrow().release_guards[0], + TileGuard::Scalar(true), + "the late subscriber inherits the release rather than starting at nothing" + ); + drop(late); + } /// A `Memo` releases its input universally as soon as the input hands over a /// complete tile, and from then on the cache is the value: repeated pulls diff --git a/src/interpreter/tile_operators/iterate.rs b/src/interpreter/tile_operators/iterate.rs index ef7f162e5..be0d3446a 100644 --- a/src/interpreter/tile_operators/iterate.rs +++ b/src/interpreter/tile_operators/iterate.rs @@ -6,7 +6,7 @@ use super::*; use crate::ccl::TagMap; use crate::interpreter::{ BaseType, ColumnValue, Consumer, Extent, NotifyOrSubscribeResult, Scheduler, SharedConsumer, - UnionArm, Value, forwarding_consumer, + UnionArm, Value, scheduler::shared_consumer, }; /// Produces a sealed-function tile whose domain and codomain both equal `extent`. @@ -36,7 +36,7 @@ impl IterateExtent { ) { match extent { Extent::DataSourceDomain(extent_impl, ..) => { - scheduler.add_source_handle(extent_impl.clone(), forwarding_consumer(&consumer)); + scheduler.add_source_handle(extent_impl.clone(), Rc::downgrade(&consumer)); } Extent::Record(fields) => { for field_extent in fields.values() { @@ -72,6 +72,7 @@ impl TileOperator for IterateExtent { base: ProducerBase::new(IterateExtentProducer::alloc_id(), &self.tiling), extent: self.extent.clone(), released: Predicate::False, + source_wakeup: None, }); let NotifyOrSubscribeResult { notify, subscribe } = @@ -80,10 +81,12 @@ impl TileOperator for IterateExtent { consumer.notify(); } if subscribe { - let consumer_wrapper = Rc::new(RefCell::new(move || { - consumer.notify(); - })); - Self::add_all_source_handles(&self.extent, consumer_wrapper, scheduler); + // The producer owns the registration: the scheduler holds only a + // `Weak`, so this handle is what keeps the source waking this + // producer, and dropping the producer deregisters it. + let consumer_wrapper = shared_consumer(consumer); + Self::add_all_source_handles(&self.extent, consumer_wrapper.clone(), scheduler); + producer.source_wakeup = Some(consumer_wrapper); let name = producer.name(); // Register this producer with any data sources in the extent by calling release with // a false predicate. This way the sources knows about all producers that read it @@ -116,6 +119,15 @@ struct IterateExtentProducer { /// be safely shrunk (shrinking source2's key 0 would prevent future /// cross-product pairs like (1, 0) from ever being produced). released: Predicate, + /// The wake-up this producer registered with the scheduler, which holds only + /// a `Weak` to it. + /// + /// Owning it here ties the registration's lifetime to the producer's: a + /// producer carried across a program update keeps waking, and one the update + /// dropped is pruned on the next + /// [`check_for_notifications`](Scheduler::check_for_notifications). + /// `None` when the extent needs no source subscription. + source_wakeup: Option>>, } fn get_iterate_extent_predicate(extent: &Extent) -> Predicate { @@ -477,6 +489,7 @@ mod tests { base: ProducerBase::new(0, &tiling), extent, released: Predicate::False, + source_wakeup: None, }; let tile = producer.get(producer.tiling().universal_guard()); let Tile::SealedFunction { domain, .. } = tile else { diff --git a/src/interpreter/types/extent.rs b/src/interpreter/types/extent.rs index 62c22b663..e88124374 100644 --- a/src/interpreter/types/extent.rs +++ b/src/interpreter/types/extent.rs @@ -284,6 +284,46 @@ pub trait DataSourceDomainExtentImpl { /// Release the region described by `obsolete` for the given producer — those domain values no longer /// need to be retained by the source. fn release(&mut self, producer: &str, obsolete: Predicate); + /// Record what every current producer has released, so a producer + /// registering with this source from now on starts there rather than at the + /// oldest value it still holds. + /// + /// Called when a running program is replaced + /// ([`LiveProgram::update`](crate::live_program::LiveProgram::update)). The + /// operators the replacement rebuilds register as new producers, and a source + /// hands a newly-registered one everything it has retained, so without this + /// the replacement recomputes the program's history instead of continuing it + /// and re-emits an output for every input the replaced version answered. + /// + /// What is carried is the *agreed* release — the part every producer is + /// finished with — so an element that arrived but went unhandled is still + /// delivered to whoever takes over. + /// + /// There is no default: a source that silently did not carry would replay its + /// history into the replacement, which reads as the program answering every + /// request it had already answered. + fn carry_release_to_new_producers(&mut self); + + /// The first position a producer registering with this source from now on + /// will be offered. + /// + /// A store built over this source starts here rather than at `0`: the source + /// will never offer the positions below it, and a drive based lower waits for + /// an element that is not coming. `0` for a source that has released nothing, + /// which is every source of a program's first version, so this is `0` + /// wherever there is no predecessor to have advanced it. + /// + /// The exception is a store resuming a predecessor over this same source, + /// which starts at *its* frontier instead — deliberately behind this, because + /// a drive reads one position back through its input and the source still owes + /// the replacement an element the recurrence has already decided. + /// + /// There is no default, for the same reason + /// [`carry_release_to_new_producers`](Self::carry_release_to_new_producers) + /// has none: a source that answered `0` when it had advanced would base a + /// drive below every position it will offer, and the drive would wait for an + /// element that is not coming. + fn first_position_for_a_new_producer(&self) -> usize; } impl PartialEq for dyn DataSourceDomainExtentImpl { diff --git a/src/lib.rs b/src/lib.rs index a4ed798f3..71f63f981 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,9 @@ pub mod ccl; pub mod chl_parser; +pub mod control_port; pub mod interpreter; +pub mod live_program; pub mod pretty_graph; pub mod pretty_tree; pub mod util; diff --git a/src/live_program.rs b/src/live_program.rs new file mode 100644 index 000000000..899207650 --- /dev/null +++ b/src/live_program.rs @@ -0,0 +1,238 @@ +//! Replacing a running program with a new version of its source. +//! +//! A [`LiveProgram`] is a compiled program plus the operation that swaps it for +//! another. The new version inherits the running one's sources and sinks and +//! whichever of its operators compute the same thing; everything else is rebuilt. +//! +//! # What an update may change +//! +//! Its logic freely, and its sources and sinks by addition: a version may open a +//! source or a sink the running program does not have and serves it as soon as +//! the swap completes, and one it stops serving is retired. What it may not do is +//! break the continuity of state — see [`update`](LiveProgram::update). +//! +//! # What survives it +//! +//! Every `Let` binding and every `Transact` store whose computation is +//! unchanged, together with what that computation has accumulated. Conversion +//! identifies each by the α-invariant +//! [`resolved_hash`](crate::ccl::content_hash::resolved_hash) of the term it +//! realizes and adopts the operator behind a matching one, so an accumulator the +//! update did not touch keeps its accumulation. Reuse is hereditary: an operator +//! is adopted only when every binding it reads was adopted too, so a +//! carried-forward operator is never left reading a subgraph the update rebuilt. +//! How much an update reuses does not depend on how many updates preceded it. +//! +//! A variable whose logic *was* edited keeps its value too. Its store is rebuilt, +//! and each variable it still declares is seeded from what the retired version +//! left it holding, so the new rule governs from the swap onwards without +//! discarding what came before. +//! +//! A binding compiled under an iteration is rebuilt regardless. Its operator is +//! parameterized by an iteration input that is not part of the term, so the term +//! does not identify it. + +use std::sync::mpsc; + +use crate::ccl::{ + context::{CompileError, CompiledProgram, GlobalContext, Phase, compile_program}, + diff::diff, +}; +use crate::interpreter::{ + Consumer, + operator_conversion::{ReuseTally, StateConflict}, + tile_operators::TileProducer, +}; + +/// Builds the consumer that wakes the driver for a program's `main` output. +/// +/// Called once per version: each compilation subscribes its own. +pub type MainConsumerFactory<'a> = &'a dyn Fn() -> Box; + +/// A compiled program being driven, and the version-swap operation over it. +pub struct LiveProgram { + program: CompiledProgram, + /// The `main` output's producer, held out of `program.outputs` so the other + /// outputs stay borrowable while the driver pulls it. + main_producer: Option>, +} + +/// What one accepted [`LiveProgram::update`] did. +pub struct UpdateReport { + /// The rendered difference between the two versions. + pub diff: String, + /// How much of the replaced version the new one adopted. + pub reuse: ReuseTally, +} + +impl LiveProgram { + /// Compile `code` and subscribe it. + pub fn start( + ctx: &mut GlobalContext, + code: &str, + main_consumer: MainConsumerFactory<'_>, + ) -> Result> { + let mut program = compile_program(ctx, code, main_consumer())?; + let main_producer = program.main_mut().and_then(|o| o.producer.take()); + Ok(LiveProgram { + program, + main_producer, + }) + } + + /// The compiled program. + pub fn program(&self) -> &CompiledProgram { + &self.program + } + + /// The `main` output's producer, for inspection. + pub fn main_producer(&self) -> Option<&dyn TileProducer> { + self.main_producer.as_deref() + } + + /// The `main` output's producer, for a driver to pull. + pub fn main_producer_mut(&mut self) -> Option<&mut Box> { + self.main_producer.as_mut() + } + + /// Whether this program has a `main` output to drive. + pub fn has_main(&self) -> bool { + self.main_producer.is_some() + } + + /// Fires once every sink output has reached a terminal tile. + pub fn done(&self) -> &mpsc::Receiver<()> { + &self.program.done + } + + /// The source this version was compiled from. + pub fn source(&self) -> &str { + &self.program.source + } + + /// Render how `code` differs from this version, comparing at `phase`. + /// + /// Compiles both sides against the running sources and sinks, which opens + /// nothing and leaves the running program untouched: a route the registry does + /// not hold is named rather than opened + /// ([`Endpoints::Inherited`](crate::ccl::lower::Endpoints::Inherited)). + /// Compiling the new version in a fresh [`GlobalContext`] instead would try to + /// bind a port this program already holds. + pub fn diff_against( + &self, + ctx: &GlobalContext, + code: &str, + phase: Phase, + ) -> Result> { + let old = ctx.sources_and_sinks().compile_to(self.source(), phase)?; + let new = ctx.sources_and_sinks().compile_to(code, phase)?; + let d = diff(&old, &new); + if d.is_identical() { + return Ok(format!("no difference at phase {phase:?}\n")); + } + Ok(format!( + "phase {phase:?}: {} divergence(s), {} shared root(s)\n\n{d}", + d.divergences().len(), + d.shared_roots().len(), + )) + } + + /// Replace this program with the version `code` describes. + /// + /// Rejected when the new version cannot **take over the state**: every + /// mutable variable the running program holds a value for must be one the + /// new version still declares, at the same type, so that its value is seeded + /// rather than discarded. Those are the two refusals [`StateConflict`] names. + /// Everything else is allowed: logic freely, sources and sinks by addition, + /// and a variable moving to another loop or into a transaction — it seeds with + /// the value it held and counts positions in whatever it now iterates. + /// + /// That is the whole guard, and it is narrower than it first looks. A + /// version that adds an `http_serve` route serves it as soon as the swap + /// completes, and one that stops serving a route retires it, so that address + /// answers 404 rather than hanging. Only a value with nowhere to be seeded + /// from is refused, because that is the one outcome an author cannot see + /// having happened: the program carries on answering and only the + /// accumulated history is gone. + /// + /// On `Err` the running program is untouched and still serving: both the + /// compile to [`Phase::Planning`] and this check run before anything is torn + /// down. + /// + /// The guard's tree is compiled separately from the one that gets built. + /// `run_frontend` goes from source to a stop phase and nothing continues a + /// stopped tree into operator conversion, so checking before tearing down + /// means compiling twice — see `src/ccl/design/live-update.md`, "Order of an + /// update". + /// + /// # Panics + /// + /// Panics if the real compile fails after the [`Phase::Planning`] one + /// succeeded. The two run the same passes over the same sources and sinks, so + /// disagreement is a compiler bug — and one that has already taken the program down, which is + /// not a state to hand back to a caller as a rejection. + pub fn update( + &mut self, + ctx: &mut GlobalContext, + code: &str, + main_consumer: MainConsumerFactory<'_>, + ) -> Result> { + let diff = self.diff_against(ctx, code, Phase::AsOfRead)?; + let planned = ctx.sources_and_sinks().compile_to(code, Phase::Planning)?; + + // What the new version can take over is read off its planned tree, + // before anything is built from it, so a version that would lose a value + // or change its type is refused while the running program is whole. + let conflicts = ctx.state_conflicts(&planned); + if !conflicts.is_empty() { + let mut lines: Vec = conflicts + .iter() + .map(|c| { + let what = c.path(); + match c { + StateConflict::Dropped { .. } => { + format!("{what} is no longer declared") + } + StateConflict::Retyped { held, declared, .. } => { + format!("{what} is now {declared} rather than {held}") + } + } + }) + .collect(); + lines.sort(); + return Err(vec![CompileError::Unsupported(format!( + "this version cannot take over state the running program is holding: {}. \ +A value carries forward only into the same variable, at the same type.", + lines.join(", "), + ))]); + } + + self.tear_down(); + ctx.retire_version(); + let next = LiveProgram::start(ctx, code, main_consumer) + .expect("a version that compiled to Planned must compile to operators"); + *self = next; + Ok(UpdateReport { + diff, + reuse: ctx.reuse(), + }) + } + + /// Drop this version's operator graph. + /// + /// Detaching the sinks is what ends this version's dispatch, and dropping + /// the outputs alone does not achieve it: an operator the next version + /// carries forward still holds the notification closure that reaches this + /// version's sink consumers, so they would keep being woken and keep writing + /// to sinks the next version now owns + /// ([`SinkConsumer::detach`](crate::interpreter::SinkConsumer::detach)). + fn tear_down(&mut self) { + self.main_producer = None; + for output in &self.program.outputs { + if let Some(consumer) = &output.sink_consumer { + consumer.borrow_mut().detach(); + } + } + self.program.outputs.clear(); + } +} diff --git a/src/main.rs b/src/main.rs index 3d91db9a5..2e8415763 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,38 +1,99 @@ -use std::{thread, time::Duration}; +use std::{cell::RefCell, rc::Rc, thread, time::Duration}; use cambra::{ ccl::{ - context::{GlobalContext, compile_program, eprint_errors}, + context::{GlobalContext, ReuseTally, eprint_errors, render_errors}, symbolic::symbolic, }, + control_port::{ControlPort, ControlReply, ControlRequest}, interpreter::{ Consumer, tile_operators::{FunctionGuard, Tile, TileGuard}, }, + live_program::LiveProgram, pretty_graph::pretty_tile_operator, web_inspector::WebInspector, }; use log::debug; +/// Render the running program's producers into the inspector's snapshot. +fn snapshot(live: &LiveProgram, inspector: Option<&WebInspector>, tick: u64) { + let Some(inspector) = inspector else { return }; + inspector.update_snapshot(tick, |add| { + // The `main` producer is held out of the compiled outputs for the + // driver, so it is read off the program rather than found among them. + if let Some(p) = live.main_producer() { + add(p); + } + for output in live.program().sinks() { + if let Some(c) = &output.sink_consumer { + c.borrow().with_producer(|p| add(p)); + } + } + }); +} + +/// Service at most one pending control request. +/// +/// One request per call rather than draining the queue: an accepted `/update` +/// replaces the program, so the requests behind it would be answered against a +/// version that no longer exists. +fn poll_control( + control: Option<&ControlPort>, + ctx: &mut GlobalContext, + live: &mut LiveProgram, + main_consumer: &dyn Fn() -> Box, + new_data: &Rc>, +) { + let Some(port) = control else { return }; + let Some(message) = port.poll() else { return }; + let reply = match message.request() { + ControlRequest::Diff { code, phase } => match live.diff_against(ctx, code, *phase) { + Ok(rendered) => ControlReply::ok(rendered), + Err(errs) => ControlReply::rejected(render_errors(&errs, "", code)), + }, + ControlRequest::Update { code } => match live.update(ctx, code, main_consumer) { + Ok(report) => { + // The new graph has subscribed but nothing has pulled it, so arm + // the driver for one pass. + *new_data.borrow_mut() = true; + let ReuseTally { adopted, bound } = report.reuse; + ControlReply::ok(format!( + "updated: {adopted}/{bound} bindings adopted\n\n{}", + report.diff + )) + } + Err(errs) => ControlReply::rejected(render_errors(&errs, "", code)), + }, + }; + message.answer(reply); +} + /// Runs a Cambra program from a source string. /// /// `src_name` is the label shown in error reports (typically the input /// file name). Returns `Err(())` if compilation failed; the errors have /// already been rendered to stderr by [`eprint_errors`], so the caller's /// job is just to exit non-zero. -fn run_program(src_name: &str, code: &str, inspect_port: Option) -> Result<(), ()> { - use std::{cell::RefCell, rc::Rc}; - +fn run_program( + src_name: &str, + code: &str, + inspect_port: Option, + control_port: Option, +) -> Result<(), ()> { let new_data = Rc::new(RefCell::new(false)); - let new_data_clone = new_data.clone(); - let consumer: Box = Box::new(move || { - debug!("Main loop received notification"); - *new_data_clone.borrow_mut() = true; - }); + let flag = new_data.clone(); + let main_consumer = move || -> Box { + let flag = flag.clone(); + Box::new(move || { + debug!("Main loop received notification"); + *flag.borrow_mut() = true; + }) + }; let mut ctx = GlobalContext::default(); - let mut compiled = match compile_program(&mut ctx, code, consumer) { - Ok(c) => c, + let mut live = match LiveProgram::start(&mut ctx, code, &main_consumer) { + Ok(p) => p, Err(errs) => { eprint_errors(&errs, src_name, code); return Err(()); @@ -42,85 +103,82 @@ fn run_program(src_name: &str, code: &str, inspect_port: Option) -> Result< let inspector = inspect_port.map(|port| { // Render every output's operator tree. The AST shown is the full // join-planned program (shared across all outputs). - let op_parts: Vec = compiled + let op_parts: Vec = live + .program() .outputs .iter() .map(|o| pretty_tile_operator(o.op.as_ref())) .collect(); - WebInspector::new(port, symbolic(&compiled.ast), op_parts.join("\n\n")) + WebInspector::new(port, symbolic(&live.program().ast), op_parts.join("\n\n")) }); - - // Pull the main producer out of `compiled` so the rest of the outputs can - // be borrowed immutably during snapshot() while we drive the producer. - let mut main_producer = compiled.main_mut().and_then(|o| o.producer.take()); - - let snapshot = - |tick: u64, - main_producer: Option<&dyn cambra::interpreter::tile_operators::TileProducer>| { - if let Some(ref insp) = inspector { - insp.update_snapshot(tick, |add| { - if let Some(p) = main_producer { - add(p); - } - for output in compiled.sinks() { - if let Some(ref c) = output.sink_consumer { - c.borrow().with_producer(|p| add(p)); - } - } - }); - } - }; + let control = control_port.map(ControlPort::new); let mut tick = 0u64; // Drive the `main` output (if any) until it signals a universal release. // For sink-only programs this loop is skipped entirely. - if let Some(producer) = main_producer.as_mut() { - loop { - while !*new_data.borrow() { - ctx.scheduler().check_for_notifications(); - } - *new_data.borrow_mut() = false; - - debug!("Main calling get"); - let tile = producer.get(producer.tiling().universal_guard()); - snapshot(tick, Some(producer.as_ref())); - tick += 1; + while live.has_main() { + while !*new_data.borrow() { + ctx.scheduler().check_for_notifications(); + poll_control( + control.as_ref(), + &mut ctx, + &mut live, + &main_consumer, + &new_data, + ); + } + *new_data.borrow_mut() = false; - let release_guard = match &tile { - Tile::Scalar(cv) => TileGuard::Scalar(!cv.is_empty()), - Tile::SealedFunction { - domain_predicate, .. - } => TileGuard::Function(FunctionGuard::Domain(domain_predicate.clone())), - other => panic!("Unexpected top-level tile shape: {other:?}"), - }; - debug!("Main releasing with {release_guard:?}"); - let done = release_guard.is_universal(); - producer.release(release_guard); - // Producers can return empty tiles, but still have more data. - let is_empty = match &tile { - Tile::Scalar(cv) => cv.is_empty(), - Tile::SealedFunction { domain, .. } => domain.is_empty(), - _ => false, - }; - if !is_empty || done { - println!("Got value: {tile:#?}"); - } - if done { - break; - } + // Re-read the producer each pass: an update between ticks replaces it. + let Some(producer) = live.main_producer_mut() else { + break; + }; + debug!("Main calling get"); + let tile = producer.get(producer.tiling().universal_guard()); + + let release_guard = match &tile { + Tile::Scalar(cv) => TileGuard::Scalar(!cv.is_empty()), + Tile::SealedFunction { + domain_predicate, .. + } => TileGuard::Function(FunctionGuard::Domain(domain_predicate.clone())), + other => panic!("Unexpected top-level tile shape: {other:?}"), + }; + debug!("Main releasing with {release_guard:?}"); + let done = release_guard.is_universal(); + producer.release(release_guard); + snapshot(&live, inspector.as_ref(), tick); + tick += 1; + // Producers can return empty tiles, but still have more data. + let is_empty = match &tile { + Tile::Scalar(cv) => cv.is_empty(), + Tile::SealedFunction { domain, .. } => domain.is_empty(), + _ => false, + }; + if !is_empty || done { + println!("Got value: {tile:#?}"); + } + if done { + break; } } // If there are sinks, keep the scheduler running until they all signal // completion. Long-lived servers (e.g. http_serve) never signal, so this // loop runs until the process exits. - if compiled.sinks().next().is_some() { + if live.program().sinks().next().is_some() { loop { ctx.scheduler().check_for_notifications(); - snapshot(tick, None); + snapshot(&live, inspector.as_ref(), tick); tick += 1; - if compiled.done.try_recv().is_ok() { + poll_control( + control.as_ref(), + &mut ctx, + &mut live, + &main_consumer, + &new_data, + ); + if live.done().try_recv().is_ok() { break; } // TODO we shouldn't need to sleep here; we should come up with a better interface @@ -139,25 +197,30 @@ fn main() { let mut input_file = None; let mut inspect_port: Option = None; + let mut control_port: Option = None; for arg in &args[1..] { if arg == "--inspect" { inspect_port = Some(8080); } else if let Some(port_str) = arg.strip_prefix("--inspect=") { inspect_port = Some(port_str.parse().expect("Invalid port for --inspect")); + } else if arg == "--control" { + control_port = Some(8081); + } else if let Some(port_str) = arg.strip_prefix("--control=") { + control_port = Some(port_str.parse().expect("Invalid port for --control")); } else { input_file = Some(arg.clone()); } } let input_file = input_file.unwrap_or_else(|| { - eprintln!("Usage: cambra [--inspect[=PORT]] "); + eprintln!("Usage: cambra [--inspect[=PORT]] [--control[=PORT]] "); std::process::exit(1); }); let code = std::fs::read_to_string(&input_file).expect("Failed to read input file"); - if run_program(&input_file, &code, inspect_port).is_err() { + if run_program(&input_file, &code, inspect_port, control_port).is_err() { std::process::exit(1); } @@ -176,6 +239,6 @@ mod tests { #[test] fn test_run_program() { - run_program("", "x = 1; x", None).unwrap(); + run_program("", "x = 1; x", None, None).unwrap(); } } diff --git a/src/util.rs b/src/util.rs index 307428f2a..7dc416828 100644 --- a/src/util.rs +++ b/src/util.rs @@ -171,6 +171,14 @@ impl ScopeStack { self.scopes.push(HashMap::new()); } + /// Every binding in the stack, outermost scope first and innermost last. + /// + /// Order within one scope is unspecified — a scope is a `HashMap`, and two + /// bindings in one scope cannot share a key. + pub fn iter_bindings(&self) -> impl Iterator { + self.scopes.iter().flat_map(|s| s.iter()) + } + /// Pop the innermost scope. /// /// Panics on underflow (unless already panicking). Prefer the drop of a diff --git a/tests/compilation_pipeline/transactions.rs b/tests/compilation_pipeline/transactions.rs index 5bdb1eb6d..ee9644297 100644 --- a/tests/compilation_pipeline/transactions.rs +++ b/tests/compilation_pipeline/transactions.rs @@ -1676,10 +1676,7 @@ fn a_cross_domain_read_coexists_with_a_second_store() { b := b + y (await_final(a), await_final(b)) "#}; - assert_eq!( - commit_stores(code), - vec!["[0, 1][acc0]", "Txn[a]", "Txn[b]"] - ); + assert_eq!(commit_stores(code), vec!["[0, 1][cnt]", "Txn[a]", "Txn[b]"]); check_tile( code, Tile::Record(std::collections::HashMap::from([ @@ -1739,10 +1736,7 @@ fn a_cross_domain_accumulator_depending_on_an_await_nests_inside_that_store() { b := b + acc await_final(b) "#}; - assert_eq!( - commit_stores(code), - vec!["Txn[a]", "[0, 1][acc0]", "Txn[b]"] - ); + assert_eq!(commit_stores(code), vec!["Txn[a]", "[0, 1][acc]", "Txn[b]"]); check_tile(code, Tile::Scalar(ColumnValue::Ints(vec![92]))); } diff --git a/tests/programs/common/mod.rs b/tests/programs/common/mod.rs index b49a2066f..835035e05 100644 --- a/tests/programs/common/mod.rs +++ b/tests/programs/common/mod.rs @@ -79,6 +79,7 @@ use cambra::{ ColumnValue, Consumer, FuncBinding, Tile, Value, bindings_are_list, tile_operators::scalar_tile_to_column_value, }, + live_program::LiveProgram, }; // --------------------------------------------------------------------------- @@ -245,6 +246,19 @@ pub fn compile_sink(source: &str) -> GlobalContext { ctx } +/// Start `source` as a [`LiveProgram`], for a test that replaces it with +/// another version. +/// +/// [`compile_sink`] is the right helper when a test only runs one version. This +/// one hands back the program, which [`LiveProgram::update`] needs and which the +/// caller must keep alive alongside the context. +pub fn start_sink(source: &str) -> (GlobalContext, LiveProgram) { + let mut ctx = GlobalContext::default(); + let program = LiveProgram::start(&mut ctx, source, &|| Box::new(|| {})) + .unwrap_or_render("", source); + (ctx, program) +} + /// Port allocation for the `{PORT}` placeholder in sink programs. Lives in the /// library, behind `test-helpers`, so this crate and `tests/http_server.rs` /// share one implementation — see [`reserve_test_port`] for why the naive @@ -274,7 +288,21 @@ pub fn http_post(port: u16, path: &str, body: &str) -> String { raw_http(port, &request) } -fn raw_http(port: u16, request: &str) -> String { +pub fn raw_http(port: u16, request: &str) -> String { + raw_http_response(port, request) + .split_once("\r\n\r\n") + .map(|(_, body)| body.to_string()) + .unwrap_or_default() +} + +/// Send `request` and return the whole response, status line and headers +/// included. +/// +/// [`raw_http`] answers with the body, which is what a test about a program's +/// output wants. Use this one where the status code is the contract — the control +/// port distinguishes an accepted request from a rejected one by status, and a +/// body-only reading cannot tell them apart. +pub fn raw_http_response(port: u16, request: &str) -> String { let mut stream = TcpStream::connect(format!("127.0.0.1:{port}")).expect("failed to connect to test server"); stream @@ -285,9 +313,7 @@ fn raw_http(port: u16, request: &str) -> String { stream .read_to_string(&mut raw) .expect("failed to read HTTP response"); - raw.split_once("\r\n\r\n") - .map(|(_, body)| body.to_string()) - .unwrap_or_default() + raw } /// Drive `ctx`'s scheduler on the current thread until `rx` delivers a diff --git a/tests/programs/live_update/mod.rs b/tests/programs/live_update/mod.rs new file mode 100644 index 000000000..7cbffa044 --- /dev/null +++ b/tests/programs/live_update/mod.rs @@ -0,0 +1,2101 @@ +//! Replacing a running program with a new version of its source, against a +//! program that is actually running. +//! +//! # The programs +//! +//! The gallery entry is one program and the version that replaces it: +//! `program.cambra` is a guestbook where `POST /sign` accumulates into a mutable +//! variable and `GET /peek` holds no state, and `updated.cambra` is the same +//! program with the accumulating loop edited. That pair is what a reader should +//! look at to see what an update *is*. +//! +//! Everything else the cases drive is scaffolding, and lives inline in +//! [`fixtures`]: variants that differ from a base by the single edit their case is +//! about, plus the shapes with no single base — a program on two ports, and the +//! `stdin`-sourced ones. The bases they vary: +//! +//! | Base | Shape | +//! | --- | --- | +//! | `guestbook` | The gallery program. Both its loops fall in one causal group, so one `Transact` store carries them. | +//! | `two-loops` | `POST /a` and `POST /b` each accumulate into their own variable. Independent, so a store each — one stays adoptable while the other is rebuilt. | +//! | `two-accumulators` | One loop carrying two variables (`left` and `right`), for the cases about telling them apart. | +//! | `one-stateful-loop` | `POST /p` accumulates, `POST /q` does not — the pair a variable can move between. | +//! | `latest-write` | A transactional variable (`Mut(String, Txn)`) that `POST /set` overwrites and `GET /get` reads. | +//! | `running-log` | A transactional variable that `POST /set` *appends* to, so every commit leaves a mark a replay would show. | +//! | `two-transactions` | Two transactional variables written and read from disjoint endpoint pairs, so they fall in different causal groups and each gets its own commit store. | +//! +//! The `stdin` cases drive the binary as a subprocess +//! ([`launch_under_control`]), because a `main` output fed by `stdin` belongs to +//! the binary's own loop rather than to a sink a test can pump. Those are also the +//! only cases that reach the control port over HTTP; every other case calls +//! `LiveProgram` directly. +//! +//! One case — `a_fold_interrupted_partway_resumes_at_the_position_it_reached` — +//! pulls a program's value itself instead. A fold over a fixed collection needs no +//! source, so pulling it is what makes the position an update lands on nameable: +//! one pull decides one position, where a socket lands wherever the notification +//! round it arrives in reaches. +//! +//! # What an update may do +//! +//! | Change | Expected | +//! | --- | --- | +//! | Logic outside a store's recurrence | Accepted; the store is adopted and its variables are untouched | +//! | Logic inside one | Accepted; the store is rebuilt and each variable resumes from the value it held, so what was recorded stands and the new rule governs from here | +//! | An edit to one of two independent loops | Accepted; the other's store is adopted | +//! | A loop gains an accumulator | Accepted; the others resume, the new one starts at its init | +//! | A variable moves to another loop, or to or from a transaction | Accepted; it seeds with the value it held and decides its new loop's positions from `0` | +//! | A loop reads another source — a port change, say | Accepted; same as above, and the port it left is released | +//! | Two loops swap which source they read | Accepted; each keeps its value and continues where its new source has got to | +//! | A variable moves to a loop over a fixed collection | Accepted; same as above — the value seeds and the collection folds on top of it | +//! | The body of a loop over a fixed collection is edited | Accepted; the fold resumes at the position it had reached, so the new rule governs the elements left | +//! | The same, with the fold caught partway | Accepted; every element is folded once, and the cut falls at the position the retired version had reached | +//! | The collection itself is edited | Accepted; the new collection counts in its own sequence, so it is folded whole | +//! | The same, over a collection a filter narrows | Accepted; the cut falls on a position the filter kept, which is a position of the collection it filters | +//! | An endpoint is added | Accepted; the route serves as soon as the swap completes | +//! | An endpoint is removed | Accepted; the route is retired and the address answers 404, unless it was the port's last route, in which case the port is released | +//! | Repeats and reverts | Accepted; each takes effect | +//! +//! # What it may not +//! +//! | Change | Expected | +//! | --- | --- | +//! | A variable is no longer declared | Refused, naming it | +//! | A variable's type changes | Refused, naming both types | +//! | The source does not compile | Refused | +//! +//! In every refusal the running program keeps serving. Diffing is covered +//! separately and must leave it untouched whichever phase it compares at, and +//! whatever the version it is compared against would have changed. +//! +//! # Two properties worth stating +//! +//! How much an update reuses does not depend on how many updates came before it: +//! a binding is named by what it computes, not by whether the compilation before +//! this one happened to build it. +//! +//! A rebuilt store resumes rather than restarting, and resumes at the position +//! its source has reached rather than replaying it. Most cases here drive two or +//! three requests before updating, which is not enough to exercise a resuming +//! store's indexing — `a_store_resumes_however_far_its_source_has_advanced` +//! drives six for that reason. + +use std::{ + sync::mpsc, + thread, + time::{Duration, Instant}, +}; + +use indoc::indoc; + +use cambra::{ + ccl::context::{GlobalContext, Phase, ReuseTally}, + interpreter::{Consumer, Tile, Value}, + live_program::{LiveProgram, UpdateReport}, +}; + +use super::common::{ + drive_until, http_get, http_post, raw_http, raw_http_response, reserve_test_port, start_sink, +}; + +/// Run a `stdin`-sourced program under `--control`, feeding it `before`, then +/// swapping it for `updated` and feeding it `after`. +/// +/// Driven as a subprocess because a `main` output belongs to the binary's own +/// loop, not to a sink a test can pump. Such a program is not short-lived: its +/// source is unbounded, so it keeps running and is as updatable as any other. +fn stdin_across_update( + program: &str, + updated: &str, + before: &str, + after: &str, +) -> (String, String) { + use std::io::Write; + + let mut launched = launch_under_control(program); + let control = launched.control; + let v2 = launched.program.dir.join("v2.cambra"); + std::fs::write(&v2, updated).expect("write v2"); + let mut input = launched.input.take().expect("piped stdin"); + let collected = launched.collected.clone(); + let reader = launched.reader.take().expect("reader thread"); + + writeln!(input, "{before}").expect("write before"); + input.flush().expect("flush"); + thread::sleep(Duration::from_millis(400)); + + let body = std::fs::read_to_string(&v2).expect("read v2"); + let reply = raw_http( + control, + &format!( + "POST /update HTTP/1.1\r\nHost: 127.0.0.1:{control}\r\nContent-Length: {}\r\n\ + Connection: close\r\n\r\n{body}", + body.len() + ), + ); + writeln!(input, "{after}").expect("write after"); + input.flush().expect("flush"); + thread::sleep(Duration::from_millis(400)); + drop(input); + + launched.program.wait_for_exit(Duration::from_secs(10)); + reader.join().expect("reader thread"); + let text = collected.lock().unwrap().clone(); + (reply, text) +} + +/// A program running under `--control`, with its control port already answering. +/// +/// Split out of [`stdin_across_update`] because a test that only asks the control +/// port a question needs the launch and none of the feeding. +struct Launched { + program: RunningProgram, + control: u16, + /// The program's `stdin`, for a test that feeds it. `None` once taken. + input: Option, + /// Every line the program has written, accumulated by [`reader`](Self::reader). + collected: std::sync::Arc>, + reader: Option>, +} + +/// Spawn `program` under `--control` on a reserved port and wait for that port to +/// answer. +/// +/// The wait is not optional: the program binds its control port during +/// compilation, so a request sent before then is refused rather than served. +fn launch_under_control(program: &str) -> Launched { + use std::io::{BufRead, BufReader}; + use std::process::{Command, Stdio}; + + let control = reserve_test_port(); + let dir = std::env::temp_dir().join(format!("cambra-live-{control}")); + std::fs::create_dir_all(&dir).expect("scratch dir"); + let v1 = dir.join("v1.cambra"); + std::fs::write(&v1, program).expect("write v1"); + + let mut program = RunningProgram { + child: Command::new(env!("CARGO_BIN_EXE_cambra")) + .arg(format!("--control={control}")) + .arg(&v1) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn cambra"), + dir, + }; + + let input = program.child.stdin.take().expect("piped stdin"); + let out = program.child.stdout.take().expect("piped stdout"); + let collected = std::sync::Arc::new(std::sync::Mutex::new(String::new())); + let sink = collected.clone(); + let reader = thread::spawn(move || { + for line in BufReader::new(out).lines().map_while(Result::ok) { + sink.lock().unwrap().push_str(&line); + sink.lock().unwrap().push('\n'); + } + }); + + let deadline = Instant::now() + Duration::from_secs(10); + while std::net::TcpStream::connect(("127.0.0.1", control)).is_err() { + assert!(Instant::now() < deadline, "control port never opened"); + thread::sleep(Duration::from_millis(50)); + } + + Launched { + program, + control, + input: Some(input), + collected, + reader: Some(reader), + } +} + +/// A spawned program and its scratch directory, both cleaned up on drop. +/// +/// Every assertion between the spawn and the last read can fail, and a program +/// left running holds the control port it bound. A port reservation's lock dies +/// with the process that took it, so the next run's allocator hands that port +/// out again and the bind fails. +struct RunningProgram { + child: std::process::Child, + dir: std::path::PathBuf, +} + +impl RunningProgram { + /// Give the program `within` to exit on its own once its input has closed. + /// + /// Polled rather than waited on: a program that never exits fails the test + /// here instead of hanging the run. + fn wait_for_exit(&mut self, within: Duration) { + let deadline = Instant::now() + within; + while Instant::now() < deadline { + if self.child.try_wait().expect("wait on cambra").is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + panic!("the program did not exit when its input closed"); + } +} + +impl Drop for RunningProgram { + fn drop(&mut self) { + // Both calls fail for a program that already exited, which is the + // ordinary path; reaping it is what has to happen either way. + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +/// Wait for `port` to stop accepting connections. +/// +/// Releasing a port is not synchronous with the update that stopped serving it: +/// dropping the last handle unblocks the dispatcher thread, and the socket closes +/// when that thread notices. The contract is that the port *is* released, not that +/// it is released before `update` returns. +fn assert_port_released(port: u16) { + let deadline = Instant::now() + Duration::from_secs(5); + while std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() { + assert!( + Instant::now() < deadline, + "port {port} is still accepting connections after its last route went", + ); + thread::sleep(Duration::from_millis(20)); + } +} + +/// The main-output consumer a sink-only program never uses. +fn no_main() -> Box { + Box::new(|| {}) +} + +/// The programs the cases below drive, other than the two the gallery keeps as +/// files. +/// +/// Inline because they are scaffolding rather than demonstrations: each is one +/// base program's variant differing by the single edit its case is about, and a +/// gallery directory holds a program, not a fixture set. `{PORT}` is substituted +/// by [`source`]. +mod fixtures { + use indoc::indoc; + + pub const BUMP_OVER_SOURCE: &str = indoc! {r#" + n := "" + reqs, resps = http_serve("{PORT}", "POST", "/bump") + for r in reqs: + n := n + "a" + resps << n + "\n" + "#}; + + pub const BUMP_OVER_A_FIXED_LIST: &str = indoc! {r#" + n := "" + reqs, resps = http_serve("{PORT}", "POST", "/bump") + for x in ["y", "z"]: + n := n + x + for r in reqs: + resps << n + "\n" + "#}; + + pub const BUMP_OVER_A_MARKED_LIST: &str = indoc! {r#" + n := "" + reqs, resps = http_serve("{PORT}", "POST", "/bump") + for x in ["y", "z"]: + n := n + x + "!" + for r in reqs: + resps << n + "\n" + "#}; + + pub const BUMP_OVER_ANOTHER_FIXED_LIST: &str = indoc! {r#" + n := "" + reqs, resps = http_serve("{PORT}", "POST", "/bump") + for x in ["p", "q"]: + n := n + x + for r in reqs: + resps << n + "\n" + "#}; + + pub const GUESTBOOK_ADDS_ROUTE: &str = indoc! {r#" + entries := "" + + sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") + peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + added_reqs, added_resps = http_serve("{PORT}", "GET", "/added") + + for entry in sign_reqs: + entries := entries + entry + "\n" + sign_resps << entries + + for req in peek_reqs: + peek_resps << "peek\n" + + for req in added_reqs: + added_resps << "added\n" + "#}; + + pub const GUESTBOOK_DROPS_ROUTE: &str = indoc! {r#" + entries := "" + + sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") + + for entry in sign_reqs: + entries := entries + entry + "\n" + sign_resps << entries + "#}; + + pub const GUESTBOOK_DROPS_STATE: &str = indoc! {r#" + sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") + peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + + for entry in sign_reqs: + sign_resps << entry + "\n" + + for req in peek_reqs: + peek_resps << "peek\n" + "#}; + + pub const GUESTBOOK_RETYPES_STATE: &str = indoc! {r#" + entries := 0 + + sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") + peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + + for entry in sign_reqs: + entries := entries + 1 + sign_resps << "signed\n" + + for req in peek_reqs: + peek_resps << "peek\n" + "#}; + + pub const GUESTBOOK_STATELESS_EDIT: &str = indoc! {r#" + entries := "" + + sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") + peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + + for entry in sign_reqs: + entries := entries + entry + "\n" + sign_resps << entries + + for req in peek_reqs: + peek_resps << "peek edited\n" + "#}; + + pub const LATEST_WRITE: &str = indoc! {r#" + set_reqs, set_resps = http_serve("{PORT}", "POST", "/set") + get_reqs, get_resps = http_serve("{PORT}", "GET", "/get") + + latest: Mut(String, Txn) := "(none)" + + for msg in set_reqs: + with begin(): + latest := msg + set_resps << "ok\n" + + for req in get_reqs: + with begin(): + get_resps << latest + "#}; + + pub const LATEST_WRITE_WRITER_EDIT: &str = indoc! {r#" + set_reqs, set_resps = http_serve("{PORT}", "POST", "/set") + get_reqs, get_resps = http_serve("{PORT}", "GET", "/get") + + latest: Mut(String, Txn) := "(none)" + + for msg in set_reqs: + with begin(): + latest := msg + "!" + set_resps << "ok\n" + + for req in get_reqs: + with begin(): + get_resps << latest + "#}; + + pub const ONE_STATEFUL_LOOP: &str = indoc! {r#" + n := "" + p, pr = http_serve("{PORT}", "POST", "/p") + q, qr = http_serve("{PORT}", "POST", "/q") + for x in p: + n := n + "a" + pr << n + "\n" + for y in q: + qr << "q\n" + "#}; + + pub const ONE_STATEFUL_LOOP_BOTH: &str = indoc! {r#" + n := "" + m := "" + p, pr = http_serve("{PORT}", "POST", "/p") + q, qr = http_serve("{PORT}", "POST", "/q") + for x in p: + n := n + "a" + pr << n + "\n" + for y in q: + m := m + "b" + qr << m + "\n" + "#}; + + pub const ONE_STATEFUL_LOOP_MOVED: &str = indoc! {r#" + n := "" + p, pr = http_serve("{PORT}", "POST", "/p") + q, qr = http_serve("{PORT}", "POST", "/q") + for x in p: + pr << "p\n" + for y in q: + n := n + "a" + qr << n + "\n" + "#}; + + pub const RUNNING_LOG: &str = indoc! {r#" + set_reqs, set_resps = http_serve("{PORT}", "POST", "/set") + get_reqs, get_resps = http_serve("{PORT}", "GET", "/get") + + log: Mut(String, Txn) := "" + + for msg in set_reqs: + with begin(): + log := log + msg + set_resps << "ok\n" + + for req in get_reqs: + with begin(): + get_resps << log + "#}; + + pub const RUNNING_LOG_WRITER_EDIT: &str = indoc! {r#" + set_reqs, set_resps = http_serve("{PORT}", "POST", "/set") + get_reqs, get_resps = http_serve("{PORT}", "GET", "/get") + + log: Mut(String, Txn) := "" + + for msg in set_reqs: + with begin(): + log := log + "-" + msg + set_resps << "ok\n" + + for req in get_reqs: + with begin(): + get_resps << log + "#}; + + pub const TWO_ACCUMULATORS: &str = indoc! {r#" + left := "" + right := "" + + reqs, resps = http_serve("{PORT}", "POST", "/bump") + + for x in reqs: + left := left + "a" + right := right + "B" + resps << left + "|" + right + "\n" + "#}; + + pub const TWO_ACCUMULATORS_ADDED: &str = indoc! {r#" + left := "" + right := "" + extra := "" + + reqs, resps = http_serve("{PORT}", "POST", "/bump") + + for x in reqs: + left := left + "a" + right := right + "B" + extra := extra + "c" + resps << left + "|" + right + "|" + extra + "\n" + "#}; + + pub const TWO_ACCUMULATORS_REORDERED: &str = indoc! {r#" + right := "" + left := "" + + reqs, resps = http_serve("{PORT}", "POST", "/bump") + + for x in reqs: + right := right + "B" + left := left + "a" + resps << left + "|" + right + "\n" + "#}; + + pub const TWO_LOOPS_SWAPPED: &str = indoc! {r#" + a := "" + b := "" + + a_reqs, a_resps = http_serve("{PORT}", "POST", "/a") + b_reqs, b_resps = http_serve("{PORT}", "POST", "/b") + + for y in b_reqs: + a := a + y + "\n" + b_resps << a + + for x in a_reqs: + b := b + x + "\n" + a_resps << b + "#}; + + pub const TWO_LOOPS: &str = indoc! {r#" + a := "" + b := "" + + a_reqs, a_resps = http_serve("{PORT}", "POST", "/a") + b_reqs, b_resps = http_serve("{PORT}", "POST", "/b") + + for x in a_reqs: + a := a + x + "\n" + a_resps << a + + for y in b_reqs: + b := b + y + "\n" + b_resps << b + "#}; + + pub const TWO_LOOPS_ONE_EDITED: &str = indoc! {r#" + a := "" + b := "" + + a_reqs, a_resps = http_serve("{PORT}", "POST", "/a") + b_reqs, b_resps = http_serve("{PORT}", "POST", "/b") + + for x in a_reqs: + a := a + x + "\n" + a_resps << a + + for y in b_reqs: + b := b + "* " + y + "\n" + b_resps << b + "#}; + + pub const TWO_TRANSACTIONS: &str = indoc! {r#" + set_a, ok_a = http_serve("{PORT}", "POST", "/a") + set_b, ok_b = http_serve("{PORT}", "POST", "/b") + get_a, out_a = http_serve("{PORT}", "GET", "/ga") + get_b, out_b = http_serve("{PORT}", "GET", "/gb") + + x: Mut(String, Txn) := "" + y: Mut(String, Txn) := "" + + for m in set_a: + with begin(): + x := x + m + ok_a << "ok\n" + + for r in get_a: + with begin(): + out_a << x + + for m in set_b: + with begin(): + y := y + m + ok_b << "ok\n" + + for r in get_b: + with begin(): + out_b << y + "#}; + + pub const TWO_TRANSACTIONS_ONE_WRITER_EDITED: &str = indoc! {r#" + set_a, ok_a = http_serve("{PORT}", "POST", "/a") + set_b, ok_b = http_serve("{PORT}", "POST", "/b") + get_a, out_a = http_serve("{PORT}", "GET", "/ga") + get_b, out_b = http_serve("{PORT}", "GET", "/gb") + + x: Mut(String, Txn) := "" + y: Mut(String, Txn) := "" + + for m in set_a: + with begin(): + x := x + "-" + m + ok_a << "ok\n" + + for r in get_a: + with begin(): + out_a << x + + for m in set_b: + with begin(): + y := y + m + ok_b << "ok\n" + + for r in get_b: + with begin(): + out_b << y + "#}; +} + +fn source(name: &str, port: u16) -> String { + let text = match name { + "guestbook" => include_str!("program.cambra"), + "bump-over-source" => fixtures::BUMP_OVER_SOURCE, + "bump-over-a-fixed-list" => fixtures::BUMP_OVER_A_FIXED_LIST, + "bump-over-a-marked-list" => fixtures::BUMP_OVER_A_MARKED_LIST, + "bump-over-another-fixed-list" => fixtures::BUMP_OVER_ANOTHER_FIXED_LIST, + "guestbook-adds-route" => fixtures::GUESTBOOK_ADDS_ROUTE, + "guestbook-drops-route" => fixtures::GUESTBOOK_DROPS_ROUTE, + "guestbook-drops-state" => fixtures::GUESTBOOK_DROPS_STATE, + "guestbook-retypes-state" => fixtures::GUESTBOOK_RETYPES_STATE, + "guestbook-stateful-edit" => include_str!("updated.cambra"), + "guestbook-stateless-edit" => fixtures::GUESTBOOK_STATELESS_EDIT, + "latest-write" => fixtures::LATEST_WRITE, + "latest-write-writer-edit" => fixtures::LATEST_WRITE_WRITER_EDIT, + "one-stateful-loop" => fixtures::ONE_STATEFUL_LOOP, + "one-stateful-loop-both" => fixtures::ONE_STATEFUL_LOOP_BOTH, + "one-stateful-loop-moved" => fixtures::ONE_STATEFUL_LOOP_MOVED, + "running-log" => fixtures::RUNNING_LOG, + "running-log-writer-edit" => fixtures::RUNNING_LOG_WRITER_EDIT, + "two-accumulators" => fixtures::TWO_ACCUMULATORS, + "two-accumulators-added" => fixtures::TWO_ACCUMULATORS_ADDED, + "two-accumulators-reordered" => fixtures::TWO_ACCUMULATORS_REORDERED, + "two-loops" => fixtures::TWO_LOOPS, + "two-loops-swapped" => fixtures::TWO_LOOPS_SWAPPED, + "two-loops-one-edited" => fixtures::TWO_LOOPS_ONE_EDITED, + "two-transactions" => fixtures::TWO_TRANSACTIONS, + "two-transactions-one-writer-edited" => fixtures::TWO_TRANSACTIONS_ONE_WRITER_EDITED, + other => panic!("no such program: {other}"), + }; + text.replace("{PORT}", &port.to_string()) +} + +/// Run `requests` on a client thread and pump the scheduler until they finish. +fn exchange(ctx: &mut cambra::ccl::context::GlobalContext, requests: F) -> Vec +where + F: FnOnce() -> Vec + Send + 'static, +{ + let (tx, rx) = mpsc::channel::>(); + thread::spawn(move || tx.send(requests()).unwrap()); + drive_until(ctx, &rx, Duration::from_secs(5)) +} + +/// An update replaces the edited logic and leaves the untouched logic running, +/// with everything that logic has accumulated. +/// +/// The guestbook is signed twice, `/peek` is edited, and the third signature +/// still returns all three entries. +/// +/// Both loops of this program share one causal group, so the edit rebuilds their +/// store and the entries survive by being re-derived from the requests the reused +/// source operator still holds. `an_edit_to_one_accumulator_leaves_the_other_running` +/// is the case where the store itself is adopted. +#[test] +fn an_update_keeps_the_state_of_logic_it_did_not_change() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || { + vec![ + http_post(port, "/sign", "alice: hi"), + http_post(port, "/sign", "bob: hello"), + http_get(port, "/peek"), + ] + }); + assert_eq!( + before, + vec!["alice: hi\n", "alice: hi\nbob: hello\n", "peek\n"], + ); + + let report: UpdateReport = live + .update( + &mut ctx, + &source("guestbook-stateless-edit", port), + &no_main, + ) + .expect("the new version only changes logic between existing endpoints"); + + let after = exchange(&mut ctx, move || { + vec![ + http_get(port, "/peek"), + http_post(port, "/sign", "carol: hey"), + ] + }); + assert_eq!( + after, + vec![ + // The edited binding was rebuilt. + "peek edited\n", + // The untouched one kept its accumulation across the swap. + "alice: hi\nbob: hello\ncarol: hey\n", + ], + ); + + let ReuseTally { adopted, bound } = report.reuse; + assert!( + adopted > 0 && adopted < bound, + "an edit to one of two independent bindings should adopt some and rebuild some, \ + got {adopted}/{bound}", + ); +} + +/// An edit to the accumulating loop itself takes effect. +/// +/// The regression this pins: every mutable variable of a program lives in one +/// `Transact` store bound to `__reg`, and a read of one is a projection off that +/// binding. While the store was registered outside the conversion scope, `__reg` +/// was free in every such term and hashed by its bare spelling, so `sign_resps` +/// (`__reg.to_sign_resps_0`) hashed identically however the recurrence was +/// edited — and its operator was reused against a store that no longer computed +/// what it had. The edit was accepted, reported as a divergence, and silently +/// did nothing. +#[test] +fn an_edit_to_the_accumulating_loop_takes_effect() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || { + vec![ + http_post(port, "/sign", "alice"), + http_post(port, "/sign", "bob"), + ] + }); + assert_eq!( + before, + vec![ + "alice +", + "alice +bob +" + ] + ); + + live.update(&mut ctx, &source("guestbook-stateful-edit", port), &no_main) + .expect("editing a loop body is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/sign", "carol")]); + assert_eq!( + after, + // The store is rebuilt, and resumes from the value the replaced version + // had reached: the entries it already recorded stand as they were, and + // the new rule governs from here. + vec!["alice\nbob\n- carol\n"], + "the new loop body must govern the response", + ); +} + +/// A store resumes correctly however far its source has advanced. +/// +/// The regression this pins: the writer body is fed through a buffer this store +/// appends to, so a decision is indexed by the row that produced it — the *n*th +/// position *this store* drove. A store resuming a running program starts at the +/// source's frontier rather than at `0`, so looking a decision up by absolute +/// position found nothing and the drive stalled, silently: the update was +/// accepted, the program's other endpoints kept serving, and the resumed loop +/// answered nothing. +/// +/// Six prior requests rather than the two or three the cases above use. That is +/// the whole point of this case: at one or two the row index and the absolute +/// position coincide often enough for the drive to stumble through, so the rest +/// of this suite passed throughout. +#[test] +fn a_store_resumes_however_far_its_source_has_advanced() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || { + (1..=6) + .map(|i| http_post(port, "/sign", &format!("e{i}"))) + .collect() + }); + assert_eq!(before.len(), 6); + assert_eq!(before[5], "e1\ne2\ne3\ne4\ne5\ne6\n"); + + live.update(&mut ctx, &source("guestbook-stateful-edit", port), &no_main) + .expect("editing a loop body is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/sign", "e7")]); + assert_eq!( + after, + // Six entries as they were recorded, and the seventh under the new rule. + vec!["e1\ne2\ne3\ne4\ne5\ne6\n- e7\n"], + ); +} + +/// Two accumulators of one loop keep their own values when the loop is +/// rewritten with them in the other order. +/// +/// The regression this pins: a write set used to reach the store as an +/// unlabelled tuple, so an accumulator was known downstream only by its position +/// within its loop. Reordering two left both positions occupied and pointing at +/// each other, and resuming from them wrote each variable's history into the +/// other — `aaa|BBB` came back as `BBBa|aaaB`, which no assertion about a single +/// accumulator could have caught. +#[test] +fn reordering_two_accumulators_does_not_cross_their_state() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-accumulators", port)); + + let before = exchange(&mut ctx, move || { + (0..3).map(|_| http_post(port, "/bump", "x")).collect() + }); + assert_eq!(before[2], "aaa|BBB\n"); + + live.update( + &mut ctx, + &source("two-accumulators-reordered", port), + &no_main, + ) + .expect("reordering two accumulators is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!( + after, + vec!["aaaa|BBBB\n"], + "each accumulator keeps its own value" + ); +} + +/// A version may add an accumulator to a loop: the ones already there resume and +/// the new one starts from its init. +/// +/// The complement of dropping one, which is refused — a variable the new version +/// introduces has no value to lose. +#[test] +fn a_loop_may_gain_an_accumulator() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-accumulators", port)); + + let before = exchange(&mut ctx, move || { + (0..2).map(|_| http_post(port, "/bump", "x")).collect() + }); + assert_eq!(before[1], "aa|BB\n"); + + live.update(&mut ctx, &source("two-accumulators-added", port), &no_main) + .expect("adding an accumulator loses nothing"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!( + after, + // The two that were there carry; the added one starts empty. + vec!["aaa|BBB|c\n"], + ); +} + +/// A variable that moves to another loop takes its value and starts counting +/// again. +/// +/// The value is the variable's; the position belongs to the collection it was +/// counted in. `n` accumulated over `/p` and now accumulates over `/q`, so it +/// seeds with what it held and decides `/q`'s positions from `0` — none of which +/// its predecessor ever read, so none is decided twice and none is skipped. +#[test] +fn a_variable_that_moves_to_another_loop_takes_its_value_and_restarts() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("one-stateful-loop", port)); + + let before = exchange(&mut ctx, move || { + vec![http_post(port, "/p", "x"), http_post(port, "/p", "x")] + }); + assert_eq!(before, vec!["a\n", "aa\n"]); + + live.update(&mut ctx, &source("one-stateful-loop-moved", port), &no_main) + .expect("`n` is still declared, at the same type"); + + let after = exchange(&mut ctx, move || { + vec![http_post(port, "/q", "x"), http_post(port, "/p", "x")] + }); + assert_eq!( + after, + vec!["aaa\n", "p\n"], + "`n` carried its `aa` into the loop it moved to", + ); +} + +/// A program that moves to another port keeps what it has accumulated. +/// +/// The whole reason a value and its position are carried separately: the new +/// source shares nothing with the old one — different route, different buffer, +/// positions from `0` — so the position cannot follow. The value can, and an +/// author moving a service to another port means to keep the guestbook. +#[test] +fn moving_a_program_to_another_port_keeps_its_state() { + let old_port = reserve_test_port(); + let new_port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", old_port)); + + let before = exchange(&mut ctx, move || { + vec![ + http_post(old_port, "/sign", "alice"), + http_post(old_port, "/sign", "bob"), + ] + }); + assert_eq!(before, vec!["alice\n", "alice\nbob\n"]); + + live.update(&mut ctx, &source("guestbook", new_port), &no_main) + .expect("`entries` is still declared, at the same type"); + + let after = exchange(&mut ctx, move || { + vec![http_post(new_port, "/sign", "carol")] + }); + assert_eq!( + after, + vec!["alice\nbob\ncarol\n"], + "the guestbook moved with the program", + ); + + // The old port served nothing after the swap, so it was released with its + // last route. + assert_port_released(old_port); +} + +/// A transactional variable survives an edit to the writer that commits it. +/// +/// The commit store is rebuilt, because the edit is inside its recurrence, and +/// resumes `latest` from the value the retired version had committed. The read +/// endpoint is untouched throughout, so a `GET` before any further write is +/// asking the resumed store directly. +/// +/// The regression this pins: the commit store published its state and was +/// adopted when unchanged, but nothing seeded a rebuilt one, so editing a +/// transactional writer silently reset the variable to its declared init. +#[test] +fn a_transactional_variable_survives_an_edit_to_its_writer() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("latest-write", port)); + + let before = exchange(&mut ctx, move || { + vec![http_post(port, "/set", "bob"), http_get(port, "/get")] + }); + assert_eq!(before, vec!["ok\n", "bob"]); + + live.update( + &mut ctx, + &source("latest-write-writer-edit", port), + &no_main, + ) + .expect("editing a transactional writer is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || { + vec![ + // Committed before the swap, so it stands as committed. + http_get(port, "/get"), + http_post(port, "/set", "carol"), + // Committed after, so the new rule governs it. + http_get(port, "/get"), + ] + }); + assert_eq!(after, vec!["bob", "ok\n", "carol!"]); +} + +/// Editing one of `two-loops`' two independent loops leaves the other's store +/// adopted, with its entries, and applies the new rule to the edited one. +#[test] +fn an_edit_to_one_accumulator_leaves_the_other_running() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-loops", port)); + + let before = exchange(&mut ctx, move || { + vec![http_post(port, "/a", "p"), http_post(port, "/b", "q")] + }); + assert_eq!(before, vec!["p\n", "q\n"]); + + live.update(&mut ctx, &source("two-loops-one-edited", port), &no_main) + .expect("editing one loop is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || { + vec![http_post(port, "/a", "r"), http_post(port, "/b", "s")] + }); + assert_eq!( + after, + vec![ + // Untouched: its store was adopted, entries and all. + "p\nr\n", + // Edited: its store was rebuilt but resumed from `q`, so the entry it + // already held stands as recorded and the new rule governs from here. + // `q` keeping its original form also shows the two loops' state does + // not collide, each being scoped by the source its loop reads. + "q\n* s\n", + ], + ); +} + +/// How much an update reuses does not depend on how many updates came before it. +/// +/// The regression this pins: while a binding's class was its identity hash when +/// adopted and a fresh value when built, a first compilation handed out classes +/// that no later one reproduced, so every binding reading another was rebuilt on +/// the first update and reuse only settled in on the second. A program is most +/// likely to be updated exactly once, which is the case that lost the most. +#[test] +fn reuse_does_not_depend_on_how_many_updates_came_before() { + let first = { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-loops", port)); + live.update(&mut ctx, &source("two-loops-one-edited", port), &no_main) + .expect("accepted") + .reuse + }; + let after_a_no_op = { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-loops", port)); + live.update(&mut ctx, &source("two-loops", port), &no_main) + .expect("accepted"); + live.update(&mut ctx, &source("two-loops-one-edited", port), &no_main) + .expect("accepted") + .reuse + }; + assert_eq!( + first, after_a_no_op, + "the same edit reused {first:?} as a program's first update and \ + {after_a_no_op:?} as its second", + ); + let ReuseTally { adopted, bound } = first; + assert!( + adopted * 2 > bound, + "editing one of two independent loops should leave most of the program \ + in place, got {adopted}/{bound}", + ); +} + +/// The control port answers `/diff` itself, at the phase the request names, and +/// rejects a phase it does not offer. +/// +/// Every other case here calls `LiveProgram::diff_against` directly, so this is +/// what covers the wire: both ways of carrying the source, the leading `phase=`, +/// and the main loop servicing a `Diff` between ticks. +#[test] +fn the_control_port_answers_a_diff_request() { + const V1: &str = "[\"> \" + line for line in stdin()]\n"; + const V2: &str = "[\">> \" + line for line in stdin()]\n"; + + let launched = launch_under_control(V1); + let control = launched.control; + + // The source is the body here, which is how a client sends a file. A space in + // a request line would end the target, so the query form has to be encoded. + let post = |query: &str, body: &str| { + raw_http( + control, + &format!( + "POST /diff{query} HTTP/1.1\r\nHost: 127.0.0.1:{control}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ), + ) + }; + + let same = post("", V1); + assert!( + same.contains("no difference"), + "the running source does not differ from itself: {same}" + ); + + let edited = post("?phase=inferred&", V2); + assert!( + edited.contains("divergence"), + "an edit is a divergence at the phase asked for: {edited}" + ); + + let bad = raw_http_response( + control, + &format!( + "POST /diff?phase=nonsense& HTTP/1.1\r\nHost: 127.0.0.1:{control}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{V1}", + V1.len() + ), + ); + assert!( + bad.starts_with("HTTP/1.1 400"), + "an unoffered phase is a rejection, not a default: {bad}" + ); + assert!( + bad.contains("channelized"), + "and the reply names the offered set: {bad}" + ); + + // The query form, which is what `percent_decode` and `split_phase_param` are + // for. `+` is a space and every other non-alphanumeric byte is escaped, so the + // program survives a request line. + let encoded: String = V1 + .trim_end() + .bytes() + .map(|b| match b { + b' ' => "+".to_string(), + b if b.is_ascii_alphanumeric() => (b as char).to_string(), + b => format!("%{b:02X}"), + }) + .collect(); + let query_form = raw_http( + control, + &format!( + "GET /diff?{encoded} HTTP/1.1\r\nHost: 127.0.0.1:{control}\r\n\ + Connection: close\r\n\r\n" + ), + ); + assert!( + query_form.contains("no difference"), + "the same source through the query string reads the same: {query_form}" + ); +} + +/// A program whose output is its `main` value rather than a sink updates too. +/// +/// Its source is `stdin`, which is unbounded, so the program keeps running and +/// the binary's own driver loop services the control port between pulls. The +/// line written before the swap is answered by the old version and the one after +/// by the new. +#[test] +fn a_main_output_program_over_stdin_updates() { + let (reply, out) = stdin_across_update( + "[\"> \" + line for line in stdin()]\n", + "[\">> \" + line for line in stdin()]\n", + "one", + "two", + ); + assert!( + reply.contains("updated"), + "the update should be accepted: {reply}" + ); + assert!( + out.contains("\"> one\""), + "the first line predates the swap: {out}" + ); + assert!( + out.contains("\">> two\""), + "the second line follows it: {out}" + ); + assert!( + !out.contains("\">> one\""), + "the swap must not reprocess the line the old version answered: {out}" + ); +} + +/// A pure element-wise transformation splits exactly at the swap: every element +/// is emitted once, by the version that was running when it arrived. +/// +/// Eight lines with the swap after the fourth. Nothing here holds state, so what +/// is being checked is the seam itself — that the stream is neither replayed +/// through the new version nor has elements dropped at the handover. +#[test] +fn an_element_wise_transformation_splits_exactly_at_the_swap() { + let (reply, out) = stdin_across_update( + "[\"A\" + line for line in stdin()]\n", + "[\"B\" + line for line in stdin()]\n", + "L1\nL2\nL3\nL4", + "L5\nL6\nL7\nL8", + ); + assert!( + reply.contains("updated"), + "the update should be accepted: {reply}" + ); + for want in ["AL1", "AL2", "AL3", "AL4", "BL5", "BL6", "BL7", "BL8"] { + assert!(out.contains(want), "missing {want} from: {out}"); + } + for unwanted in ["BL1", "BL2", "BL3", "BL4", "AL5", "AL6", "AL7", "AL8"] { + assert!( + !out.contains(unwanted), + "{unwanted} means an element crossed the seam: {out}" + ); + } +} + +/// An accumulator in a `main`-output program carries across the swap, and each +/// half of the stream is counted by the rule in force when it arrived. +/// +/// Four lines, the rule changing from `+1` to `+2` after the second: the value +/// at EOF is `6`, not `4` (the old rule throughout) and not `8` (the new one +/// applied retroactively). Nothing about a `main` output makes its state less +/// live than a sink program's — what a value like `n` here reports is decided by +/// when it is read, and reading it at the tail of the program means EOF. +#[test] +fn a_main_output_accumulator_carries_across_the_swap() { + let (reply, out) = stdin_across_update( + "n := 0\nfor line in stdin():\n n := n + 1\nn\n", + "n := 0\nfor line in stdin():\n n := n + 2\nn\n", + "a\nb", + "c\nd", + ); + assert!( + reply.contains("updated"), + "the update should be accepted: {reply}" + ); + let flat: String = out.chars().filter(|c| !c.is_whitespace()).collect(); + assert!( + flat.contains("Ints([6,],)"), + "want 1+1+2+2; the whole run was: {out}" + ); +} + +/// A `main`-output program reports its accumulator *live* when it feeds one out, +/// and the feed shows the swap taking effect mid-stream. +/// +/// `out << n` per line makes each step observable rather than only the value at +/// EOF, so the sequence `1, 2, 4, 6` is the accumulator itself: two steps of `+1`, +/// then the swap, then two of `+2` continuing from `2` rather than restarting. +#[test] +fn a_fed_accumulator_is_observable_across_the_swap() { + let (reply, out) = stdin_across_update( + "out = defer()\nn := 0\nfor line in stdin():\n n := n + 1\n out << n\nout\n", + "out = defer()\nn := 0\nfor line in stdin():\n n := n + 2\n out << n\nout\n", + "a\nb", + "c\nd", + ); + assert!( + reply.contains("updated"), + "the update should be accepted: {reply}" + ); + let flat: String = out.chars().filter(|c| !c.is_whitespace()).collect(); + // `4` is the step that can only happen if the swap resumed from `2`; a + // restart would report `2` again. + assert!(flat.contains("Ints([4,],)"), "want a step to 4: {out}"); + assert!(flat.contains("Ints([6,],)"), "want a step to 6: {out}"); +} + +/// The state guard covers a `stdin`-sourced loop, not just an `http_serve` one. +/// +/// Nothing about the guard is HTTP-specific: it reads the variables a version +/// declares off its planned tree, and `stdin` declares them the same way. +/// The value never reaches the program's output (a scalar read of an +/// accumulator over an unbounded source never finalizes), which is exactly why +/// the guard has to catch the change rather than leaving it to be noticed. +#[test] +fn the_state_guard_covers_a_stdin_sourced_loop() { + let (reply, _) = stdin_across_update( + "n := \"\"\nfor line in stdin():\n n := n + line\nn\n", + "n := 0\nfor line in stdin():\n n := n + 1\nn\n", + "a", + "b", + ); + assert!( + reply.contains("`n` is now Int"), + "the rejection should name the stdin loop's variable and its new type: {reply}" + ); +} + +/// A version may add an endpoint, and serves it as soon as the swap completes. +/// +/// The endpoint set is not frozen: a route the registry already holds is bound +/// and one it does not is opened, in a replacement exactly as in a first +/// version. The endpoints that were already there keep working, state included. +#[test] +fn an_update_may_add_an_endpoint() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || vec![http_post(port, "/sign", "alice")]); + assert_eq!(before, vec!["alice\n"]); + + live.update(&mut ctx, &source("guestbook-adds-route", port), &no_main) + .expect("adding an endpoint is allowed"); + + let after = exchange(&mut ctx, move || { + vec![ + // The added route serves. + http_get(port, "/added"), + // The endpoints that were already there are unaffected, state and all. + http_get(port, "/peek"), + http_post(port, "/sign", "bob"), + ] + }); + assert_eq!(after, vec!["added\n", "peek\n", "alice\nbob\n"]); +} + +/// A version that stops serving a route retires it, so the address answers 404. +/// +/// The listener and its routing-table entry belong to the source/sink registry +/// and outlive the version that opened them, so a version that stops binding a +/// route has to say so. Left registered, the route keeps matching requests and +/// buffering them for a reader that no longer exists, and the client waits on a +/// reply nobody will compute — this test hangs rather than fails if that +/// regresses, because the request never comes back at all. +#[test] +fn a_version_that_stops_serving_a_route_retires_it() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || vec![http_get(port, "/peek")]); + assert_eq!(before, vec!["peek\n"]); + + live.update(&mut ctx, &source("guestbook-drops-route", port), &no_main) + .expect("dropping a stateless route is allowed"); + + let after = exchange(&mut ctx, move || { + vec![ + // Retired: the dispatcher answers rather than buffering. + http_get(port, "/peek"), + // The route that stayed is unaffected. + http_post(port, "/sign", "alice"), + ] + }); + assert_eq!(after, vec!["Not Found", "alice\n"]); +} + +/// A version that declares a held variable at a different type is rejected, and +/// the running program keeps serving. +/// +/// The value cannot be the seed of a store built for another shape. Left to +/// proceed, the store is constructed around a constant of the wrong extent and +/// the process dies on the next pull (`Scalar(Strings([..])) vs Scalar(Int)`), +/// taking every endpoint with it — the update is not recoverable at that point, +/// so it has to be refused before the swap. +#[test] +fn an_update_may_not_change_the_type_of_held_state() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || vec![http_post(port, "/sign", "alice")]); + assert_eq!(before, vec!["alice\n"]); + + let errors = live + .update(&mut ctx, &source("guestbook-retypes-state", port), &no_main) + .err() + .expect("`entries` holds a String; the new version declares it an Int"); + let rendered = format!("{errors:?}"); + assert!( + rendered.contains("`entries`") && rendered.contains("Int") && rendered.contains("String"), + "the rejection should name the variable and both types: {rendered}", + ); + + let still_serving = exchange(&mut ctx, move || vec![http_post(port, "/sign", "bob")]); + assert_eq!(still_serving, vec!["alice\nbob\n"], "state intact"); +} + +/// A version that stops declaring a variable the running program is holding a +/// value for is rejected, and the running program keeps serving. +/// +/// This is the whole endpoint/state guard: dropping a value is the one outcome +/// an author cannot see having happened, since the program carries on answering +/// and only the accumulated history is gone. +#[test] +fn an_update_may_not_drop_state() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let before = exchange(&mut ctx, move || vec![http_post(port, "/sign", "alice")]); + assert_eq!(before, vec!["alice\n"]); + + let errors = live + .update(&mut ctx, &source("guestbook-drops-state", port), &no_main) + .err() + .expect("a version that stops declaring `entries` would discard its value"); + let rendered = format!("{errors:?}"); + assert!( + rendered.contains("cannot take over state") && rendered.contains("`entries`"), + "the rejection should name the variable: {rendered}", + ); + + let still_serving = exchange(&mut ctx, move || vec![http_post(port, "/sign", "bob")]); + assert_eq!(still_serving, vec!["alice\nbob\n"], "state intact"); +} + +/// A version that does not compile is rejected before the running program is +/// touched. +#[test] +fn a_rejected_update_leaves_the_program_serving() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + live.update(&mut ctx, "x = = 1", &no_main) + .err() + .expect("a syntax error is not an update"); + + let still_serving = exchange(&mut ctx, move || vec![http_get(port, "/peek")]); + assert_eq!(still_serving, vec!["peek\n"]); +} + +/// Diffing a running program against a new version opens nothing and changes +/// nothing — the endpoint set it compiles against is the one already bound. +/// +/// The naive alternative, compiling the new version in a fresh context, would +/// try to bind a port this program holds. +#[test] +fn diffing_a_running_http_program_leaves_it_untouched() { + let port = reserve_test_port(); + let (mut ctx, live) = start_sink(&source("guestbook", port)); + + let identical = live + .diff_against(&ctx, &source("guestbook", port), Phase::AsOfRead) + .expect("the running source compiles against its own endpoints"); + assert!( + identical.contains("no difference"), + "a program should not differ from itself: {identical}", + ); + + let changed = live + .diff_against( + &ctx, + &source("guestbook-stateless-edit", port), + Phase::AsOfRead, + ) + .expect("the new version compiles against the running endpoints"); + assert!( + changed.contains("divergence"), + "an edited program should report a divergence: {changed}", + ); + + let still_serving = exchange(&mut ctx, move || vec![http_get(port, "/peek")]); + assert_eq!(still_serving, vec!["peek\n"]); +} + +/// Two calls to one function, each carrying its own loop and its own accumulator, +/// keep their state apart across an update. +/// +/// Inlining clones the function body per call site, so both accumulators are the +/// same source declaration — same spelling, same lexical position, no name of +/// their own to tell them apart. Nor can anything they compute: once `step` is +/// substituted the two stores differ *only* in their writer bodies, which is +/// exactly what the edit changes. Their identities are the spelling plus an index +/// among the variables of that spelling, which is why the edit carries and the +/// two do not cross. +#[test] +fn two_instantiations_of_one_function_keep_their_accumulators_apart() { + let v1 = concat!( + "def count_by(src, step) => Int:\n", + " total := 0\n", + " for x in src:\n", + " total := total + step\n", + " total\n", + "\n", + "lines = stdin()\n", + "a = count_by(lines, 1)\n", + "b = count_by(lines, 10)\n", + "a * 1000 + b\n", + ); + let v2 = v1.replace("total + step", "total + step * 2"); + let (reply, out) = stdin_across_update(v1, &v2, "m\nn", "o\np"); + assert!( + reply.contains("updated"), + "the update should be accepted: {reply}" + ); + let flat: String = out.chars().filter(|c| !c.is_whitespace()).collect(); + assert!( + flat.contains("Ints([6060,],)"), + "want a = 2 + 2*2 and b = 20 + 2*20; seeding either from the other reads 6042: {out}" + ); +} + +/// Two causally independent transaction groups keep their state apart across an +/// update that rebuilds one of them. +/// +/// A program has one commit store per causal group, not one commit store, so `x` +/// and `y` here live in different stores sequenced by the same `Txn` domain. +/// Their frontiers differ — `x` has taken four commits and `y` one — and the +/// store rebuilt for `x` has to resume at its own, which is why the resume +/// position hangs off each variable rather than off the recurrence. +#[test] +fn two_transaction_groups_resume_at_their_own_positions() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-transactions", port)); + + let before = exchange(&mut ctx, move || { + vec![ + http_post(port, "/a", "1"), + http_post(port, "/a", "2"), + http_post(port, "/a", "3"), + http_post(port, "/a", "4"), + http_post(port, "/b", "9"), + http_get(port, "/ga"), + http_get(port, "/gb"), + ] + }); + assert_eq!( + before, + vec!["ok\n", "ok\n", "ok\n", "ok\n", "ok\n", "1234", "9"], + ); + + live.update( + &mut ctx, + &source("two-transactions-one-writer-edited", port), + &no_main, + ) + .expect("editing one group's writer declares the same variables at the same types"); + + let after = exchange(&mut ctx, move || { + vec![ + http_get(port, "/ga"), + http_get(port, "/gb"), + http_post(port, "/a", "5"), + http_post(port, "/b", "8"), + http_get(port, "/ga"), + http_get(port, "/gb"), + ] + }); + assert_eq!( + after, + vec!["1234", "9", "ok\n", "ok\n", "1234-5", "98"], + "each group stands where it stood, and the new rule governs `x` from here", + ); +} + +/// A request the retired version received but never answered is answered by the +/// replacement, once. +/// +/// The end-to-end half of the release carry: a source hands a newly registered +/// producer everything it still holds, and `retire_version` records what the +/// retired producers had released so the replacement's do not start from the +/// oldest retained element. `UIntStreamBuffer`'s unit tests pin the buffer's side +/// of that; this pins the promise a client sees, which is that arriving before +/// the swap is not a way to be dropped. +/// +/// The request is sent and left unanswered — nothing pumps the scheduler until +/// after the update — so the version that received it is gone before it could +/// have replied. +#[test] +fn a_request_that_arrived_before_the_swap_is_answered_after_it() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + let (tx, rx) = mpsc::channel::>(); + thread::spawn(move || tx.send(vec![http_post(port, "/sign", "alice")]).unwrap()); + // Long enough for the dispatcher thread to have taken the request off the + // socket. Nothing has pumped, so no operator has seen it and no reply exists. + thread::sleep(Duration::from_millis(300)); + + live.update( + &mut ctx, + &source("guestbook-stateless-edit", port), + &no_main, + ) + .expect("editing `/peek` leaves `/sign` alone"); + + let answered = drive_until(&mut ctx, &rx, Duration::from_secs(5)); + assert_eq!( + answered, + vec!["alice\n"], + "the replacement answers the request its predecessor received", + ); + + let next = exchange(&mut ctx, move || vec![http_post(port, "/sign", "bob")]); + assert_eq!( + next, + vec!["alice\nbob\n"], + "and counted it once: a replay would read `alice` twice", + ); +} + +/// Diffing against a version that stops serving a route leaves the route serving. +/// +/// A diff answers a question; only an update changes what the program serves. The +/// two compile against the same registry, so the compile that answers the +/// question must not act on the difference it finds — the route it would retire +/// belongs to the running program, and its listener is shared. +#[test] +fn diffing_against_a_version_that_drops_a_route_does_not_retire_it() { + let port = reserve_test_port(); + let (mut ctx, live) = start_sink(&source("guestbook", port)); + + let changed = live + .diff_against( + &ctx, + &source("guestbook-drops-route", port), + Phase::AsOfRead, + ) + .expect("the new version compiles against the running endpoints"); + assert!( + changed.contains("divergence"), + "dropping a route is a difference: {changed}", + ); + + let still_serving = exchange(&mut ctx, move || vec![http_get(port, "/peek")]); + assert_eq!( + still_serving, + vec!["peek\n"], + "`/peek` is still the running program's route; only an update retires it", + ); +} + +/// A port whose last route goes is released; one that keeps a route keeps serving. +/// +/// A listener outlives the version that opened it, but not the program's interest +/// in the address. While a sibling route survives on the port, a retired route's +/// address answers 404 — there is still a server there. Once nothing is +/// registered, the port itself goes, so a program that moves its endpoints around +/// over a long life does not hold every port it ever served. +#[test] +fn a_port_whose_last_route_goes_is_released() { + let kept = reserve_test_port(); + let dropped = reserve_test_port(); + let two_ports = format!( + "a_reqs, a_resps = http_serve(\"{kept}\", \"GET\", \"/a\")\n\ + b_reqs, b_resps = http_serve(\"{dropped}\", \"GET\", \"/b\")\n\ + c_reqs, c_resps = http_serve(\"{kept}\", \"GET\", \"/c\")\n\ + for r in a_reqs:\n a_resps << \"a\\n\"\n\ + for r in b_reqs:\n b_resps << \"b\\n\"\n\ + for r in c_reqs:\n c_resps << \"c\\n\"\n" + ); + let one_port = format!( + "a_reqs, a_resps = http_serve(\"{kept}\", \"GET\", \"/a\")\n\ + for r in a_reqs:\n a_resps << \"a\\n\"\n" + ); + + let (mut ctx, mut live) = start_sink(&two_ports); + assert_eq!( + exchange(&mut ctx, move || vec![ + http_get(kept, "/a"), + http_get(dropped, "/b") + ]), + vec!["a\n", "b\n"], + ); + + live.update(&mut ctx, &one_port, &no_main) + .expect("dropping routes declares no state, so it is accepted"); + + // `/c` shared the kept port, so its address is a 404 rather than a refusal: + // the listener is still there for `/a`'s sake. + let retired = exchange(&mut ctx, move || { + vec![http_get(kept, "/a"), http_get(kept, "/c")] + }); + assert_eq!(retired, vec!["a\n", "Not Found"]); + + // The other port lost its only route, so nothing is listening there. + assert_port_released(dropped); +} + +/// Every phase the control port offers is a working diff point, not only the +/// default. Driven off `OFFERED_PHASES` itself, so a phase added to the offered +/// set is covered here without anyone remembering to add it. +#[test] +fn every_offered_phase_is_a_diff_point() { + let port = reserve_test_port(); + let (ctx, live) = start_sink(&source("guestbook", port)); + + for (spelling, phase) in cambra::control_port::OFFERED_PHASES { + let rendered = live + .diff_against(&ctx, &source("guestbook-stateless-edit", port), *phase) + .unwrap_or_else(|e| panic!("diff at {spelling} failed: {e:?}")); + assert!( + rendered.contains("divergence"), + "the edit should be visible at {spelling}: {rendered}", + ); + } +} + +/// Repeated updates keep working, including switching back to a version that +/// already ran. +#[test] +fn a_program_can_be_updated_repeatedly() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("guestbook", port)); + + for (name, expected) in [ + ("guestbook-stateless-edit", "peek edited\n"), + ("guestbook", "peek\n"), + ("guestbook-stateless-edit", "peek edited\n"), + // Twice in a row: an update to the version already running is a no-op + // that must still leave it serving. + ("guestbook-stateless-edit", "peek edited\n"), + ] { + live.update(&mut ctx, &source(name, port), &no_main) + .unwrap_or_else(|e| panic!("update to {name} rejected: {e:?}")); + let served = exchange(&mut ctx, move || vec![http_get(port, "/peek")]); + assert_eq!(served, vec![expected], "after updating to {name}"); + } +} + +/// A transaction writer does not re-attempt the transactions it committed before +/// the swap. +/// +/// The regression this pins: the drive named the item it was attempting by its +/// index among the source's *offered columns* and never released a finished one, +/// so the source went on offering every request and the replacement's drive +/// started again from the first. Six commits before the update were committed a +/// second time after it, under the new rule — visible here because each append +/// leaves a mark, and invisible to a last-write-wins variable however deep the +/// history. +#[test] +fn a_transaction_writer_does_not_replay_what_it_committed() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("running-log", port)); + + let before = exchange(&mut ctx, move || { + (1..=6) + .map(|i| http_post(port, "/set", &i.to_string())) + .collect() + }); + assert_eq!(before.len(), 6); + let logged = exchange(&mut ctx, move || vec![http_get(port, "/get")]); + assert_eq!(logged, vec!["123456"]); + + live.update(&mut ctx, &source("running-log-writer-edit", port), &no_main) + .expect("editing a transactional writer is a change between existing endpoints"); + + let after = exchange(&mut ctx, move || { + vec![http_post(port, "/set", "7"), http_get(port, "/get")] + }); + // Six as they were committed, and the seventh under the new rule — not + // `123456-1-2-3-4-5-6-7`. + assert_eq!(after, vec!["ok\n", "123456-7"]); +} + +/// Two loops swapping which source they read keep their values and pick up where +/// each source has got to. +/// +/// Both variables change domain at once, so neither can resume at its +/// predecessor's frontier — and neither can start at `0` either, because both +/// sources have already delivered and released. Each starts where the source it +/// moved to will next offer a producer. +#[test] +fn two_loops_may_swap_which_source_they_read() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("two-loops", port)); + + let before = exchange(&mut ctx, move || { + vec![ + http_post(port, "/a", "1"), + http_post(port, "/a", "2"), + http_post(port, "/b", "9"), + ] + }); + assert_eq!(before, vec!["1\n", "1\n2\n", "9\n"]); + + live.update(&mut ctx, &source("two-loops-swapped", port), &no_main) + .expect("both variables are still declared, at the same types"); + + // `/a` now writes `b` and answers on `a_resps`; `/b` now writes `a`. + let after = exchange(&mut ctx, move || { + vec![http_post(port, "/a", "3"), http_post(port, "/b", "8")] + }); + assert_eq!( + after, + vec!["9\n3\n", "1\n2\n8\n"], + "each variable kept its value and continued on the source it moved to", + ); +} + +/// A loop that gains an accumulator over a source the program was already reading +/// starts where that source has got to. +/// +/// There is no predecessor to resume from — the variable is new — so this is the +/// case that has nothing carried at all and still cannot start at `0`. `/q` has +/// delivered and released a request, and a store based below that waits for an +/// element the source will not offer again. +#[test] +fn a_stateless_loop_may_gain_an_accumulator_over_an_advanced_source() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("one-stateful-loop", port)); + + let before = exchange(&mut ctx, move || { + vec![http_post(port, "/p", "x"), http_post(port, "/q", "x")] + }); + assert_eq!(before, vec!["a\n", "q\n"]); + + live.update(&mut ctx, &source("one-stateful-loop-both", port), &no_main) + .expect("`n` is unchanged and `m` is new"); + + let after = exchange(&mut ctx, move || { + vec![http_post(port, "/q", "x"), http_post(port, "/p", "x")] + }); + assert_eq!( + after, + vec!["b\n", "aa\n"], + "`m` starts empty at `/q`'s current position, and `n` carries", + ); +} + +/// A variable that moves to a loop over a fixed collection keeps its value and +/// folds the collection on top of it. +/// +/// The move is the same one a variable makes between any two loops: the value +/// seeds, and the positions restart because they are counted in something else. +/// Nothing about the new sequence being a list rather than a source changes that +/// — `n` holds what the requests built, and the list's elements follow. +#[test] +fn a_variable_that_moves_to_a_fixed_collection_keeps_its_value() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("bump-over-source", port)); + + let before = exchange(&mut ctx, move || { + vec![http_post(port, "/bump", "x"), http_post(port, "/bump", "x")] + }); + assert_eq!(before, vec!["a\n", "aa\n"]); + + live.update(&mut ctx, &source("bump-over-a-fixed-list", port), &no_main) + .expect("`n` is still declared, at the same type"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!( + after, + vec!["aayz\n"], + "the list folds on top of what the requests built", + ); +} + +/// A fold over a fixed collection resumes where its predecessor stopped, so an +/// edit inside the loop governs the elements that are left rather than replaying +/// the ones already folded. +/// +/// The version installed here appends `"!"` to every element. None is appended, +/// because the fold had already reached the end of the list: an element is +/// decided once, by whichever version was running when it came up. +#[test] +fn a_fold_over_a_fixed_collection_resumes_where_it_stopped() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("bump-over-a-fixed-list", port)); + + let before = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!(before, vec!["yz\n"]); + + live.update(&mut ctx, &source("bump-over-a-marked-list", port), &no_main) + .expect("`n` is still declared, at the same type"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!( + after, + vec!["yz\n"], + "the new rule governs the elements left, and none are", + ); +} + +/// Pull the program's value until it settles, and return it. +/// +/// A fold's value is final once the tile is terminal, which for an induction +/// store means every position of its extent is decided. +fn drive_main_to_terminal(ctx: &mut GlobalContext, live: &mut LiveProgram) -> String { + for _ in 0..500 { + ctx.scheduler().check_for_notifications(); + let producer = live + .main_producer_mut() + .expect("the program's value is `n`"); + let guard = producer.tiling().universal_guard(); + let tile = producer.get(guard); + if tile.is_terminal() { + let Tile::Scalar(column) = tile else { + panic!("a string fold's value is a scalar, got {tile:?}"); + }; + let Value::String(s) = column.index_at(0) else { + panic!("a string fold's value is a string"); + }; + return s.to_string(); + } + } + panic!("the fold never settled"); +} + +/// The elements the two mid-fold cases fold. Twenty because the point is a fold +/// caught partway, and a shorter list finishes inside the notification round that +/// starts it. +const TWENTY: &str = r#"["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t"]"#; + +/// A fold of [`TWENTY`] into `n` whose `step` is the loop body, ending in `n` as +/// the program's value. Built rather than inline because the two mid-fold cases +/// and their two edits are one program with one line varying. +fn fold_to_main(step: &str) -> String { + format!( + indoc! {r#" + n := "" + for x in {items}: + {step} + n + "#}, + items = TWENTY, + step = step, + ) +} + +/// The same fold behind `POST /read`, which replies with `n`. +fn fold_behind_a_route(step: &str, port: u16) -> String { + format!( + indoc! {r#" + n := "" + reqs, resps = http_serve("{port}", "POST", "/read") + for x in {items}: + {step} + for r in reqs: + resps << n + "\n" + "#}, + port = port, + items = TWENTY, + step = step, + ) +} + +/// Every element of `reply`, with the ones the marking version decided flagged. +/// +/// The reply is the fold's whole value, so it says which version decided which +/// element: an element the marking version folded is followed by `!`. +fn decided_by_the_new_version(reply: &str) -> Vec { + let mut out = Vec::new(); + for c in reply.trim().chars() { + if c == '!' { + *out.last_mut().expect("a marker follows an element") = true; + } else { + out.push(false); + } + } + out +} + +/// A fold caught partway resumes at the position it had reached: the elements +/// below it keep what the retired version decided, and the new rule governs the +/// rest. +/// +/// The only case that drives a program's value directly rather than through a +/// sink, because that is what makes the position the update lands on nameable: one +/// pull decides one position, so pulling `k` times and then swapping puts the cut +/// at `k - 1` rather than wherever a socket happened to be serviced. +#[test] +fn a_fold_interrupted_partway_resumes_at_the_position_it_reached() { + const PULLS: usize = 8; + let mut ctx = GlobalContext::default(); + let mut live = + LiveProgram::start(&mut ctx, &fold_to_main("n := n + x"), &no_main).expect("compiles"); + for _ in 0..PULLS { + let producer = live + .main_producer_mut() + .expect("the program's value is `n`"); + let guard = producer.tiling().universal_guard(); + let _ = producer.get(guard); + ctx.scheduler().check_for_notifications(); + } + + live.update(&mut ctx, &fold_to_main(r#"n := n + x + "!""#), &no_main) + .expect("`n` is still declared, at the same type"); + + let value = drive_main_to_terminal(&mut ctx, &mut live); + let decided = decided_by_the_new_version(&value); + assert_eq!( + decided.len(), + 20, + "every element is folded exactly once: {value}" + ); + let resumed_at = decided.iter().position(|marked| *marked); + assert_eq!( + resumed_at, + Some(PULLS - 1), + "the elements below the frontier are the retired version's, and the rest are the new one's: {value}", + ); + assert!( + decided[PULLS - 1..].iter().all(|marked| *marked), + "the new rule governs every element from the frontier on: {value}", + ); +} + +/// A version installed while a fold is partway through is pulled without waiting +/// for a source to report new data. +/// +/// An operator notifies from inside `subscribe` — an induction store does, to +/// start its loop — and a sink consumer whose producer slot is not filled yet +/// drops that notification. A first compile does not notice, because the source +/// that has data reports it as new on the next poll. A replacement does: the +/// version it replaces already took that report, so the request in flight here +/// went unanswered until another arrived. +/// +/// Where the fold is cut is a property of one notification round rather than +/// anything the language promises, so this pins the invariant — every element +/// folded once, the new rule governing a suffix — and leaves the position to +/// `a_fold_interrupted_partway_resumes_at_the_position_it_reached`. +#[test] +fn a_version_installed_mid_fold_is_pulled_without_a_new_arrival() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&fold_behind_a_route("n := n + x", port)); + + let (tx, rx) = mpsc::channel::>(); + thread::spawn(move || tx.send(vec![http_post(port, "/read", "x")]).unwrap()); + // Let the request arrive, then advance the fold with one poll — which both + // takes the source's report of new data and leaves the fold unfinished. + thread::sleep(Duration::from_millis(400)); + ctx.scheduler().check_for_notifications(); + assert!( + rx.try_recv().is_err(), + "twenty elements outlast the round that starts them, so the reply is still pending", + ); + + live.update( + &mut ctx, + &fold_behind_a_route(r#"n := n + x + "!""#, port), + &no_main, + ) + .expect("`n` is still declared, at the same type"); + + let reply = drive_until(&mut ctx, &rx, Duration::from_secs(5)); + let value = reply.first().expect("one request, one reply").clone(); + let decided = decided_by_the_new_version(&value); + assert_eq!( + decided.len(), + 20, + "every element is folded exactly once: {value}" + ); + let resumed_at = decided + .iter() + .position(|marked| *marked) + .expect("the swap lands before the last element"); + assert!( + decided[resumed_at..].iter().all(|marked| *marked), + "the new rule governs every element from the frontier on: {value}", + ); +} + +/// A fold of [`TWENTY`]'s later half into `n`, the elements reaching the loop +/// through a filter. `step` is the loop body, as in [`fold_to_main`]. +fn filtered_fold_to_main(step: &str) -> String { + format!( + indoc! {r#" + n := "" + kept = [x for x in {items} if x > "q"] + for x in kept: + {step} + n + "#}, + items = TWENTY, + step = step, + ) +} + +/// A fold whose source is filtered resumes at the position it had reached, which +/// is a position of the *unfiltered* extent. +/// +/// The elements the filter drops occupy no position in the recurrence, so the +/// positions it does decide are a subset of the collection's and are not +/// contiguous. The resume position is one of those, and the cut has to fall on it: +/// resuming at the position after the last one *decided* would skip an element +/// nothing has folded, and resuming by counting decided elements would land in the +/// wrong place entirely. +#[test] +fn a_filtered_fold_resumes_at_a_position_of_the_collection_it_filters() { + const PULLS: usize = 2; + let mut ctx = GlobalContext::default(); + let mut live = LiveProgram::start(&mut ctx, &filtered_fold_to_main("n := n + x"), &no_main) + .expect("compiles"); + for _ in 0..PULLS { + let producer = live + .main_producer_mut() + .expect("the program's value is `n`"); + let guard = producer.tiling().universal_guard(); + let _ = producer.get(guard); + ctx.scheduler().check_for_notifications(); + } + + live.update( + &mut ctx, + &filtered_fold_to_main(r#"n := n + x + "!""#), + &no_main, + ) + .expect("`n` is still declared, at the same type"); + + let value = drive_main_to_terminal(&mut ctx, &mut live); + assert_eq!( + value, "rs!t!", + "the filter keeps `r`, `s` and `t`; the first is the retired version's and \ + the rest are the new one's", + ); +} + +/// A fold over a *different* fixed collection counts in a different sequence, so +/// it folds that collection from its first element. +/// +/// `[0, 2]` is the extent of `["y", "z"]` and of `["p", "q"]` alike; resuming the +/// second fold at the first's frontier would skip both its elements. The +/// collection is named by the term that computes it for that reason, and the +/// value carries the way it does between any two loops. +#[test] +fn a_fold_over_another_fixed_collection_starts_it_from_the_beginning() { + let port = reserve_test_port(); + let (mut ctx, mut live) = start_sink(&source("bump-over-a-fixed-list", port)); + + let before = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!(before, vec!["yz\n"]); + + live.update( + &mut ctx, + &source("bump-over-another-fixed-list", port), + &no_main, + ) + .expect("`n` is still declared, at the same type"); + + let after = exchange(&mut ctx, move || vec![http_post(port, "/bump", "x")]); + assert_eq!( + after, + vec!["yzpq\n"], + "the new list is folded whole, onto the value the old one built", + ); +} diff --git a/tests/programs/live_update/program.cambra b/tests/programs/live_update/program.cambra new file mode 100644 index 000000000..662afb8c3 --- /dev/null +++ b/tests/programs/live_update/program.cambra @@ -0,0 +1,11 @@ +entries := "" + +sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") +peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + +for entry in sign_reqs: + entries := entries + entry + "\n" + sign_resps << entries + +for req in peek_reqs: + peek_resps << "peek\n" diff --git a/tests/programs/live_update/updated.cambra b/tests/programs/live_update/updated.cambra new file mode 100644 index 000000000..5b642514e --- /dev/null +++ b/tests/programs/live_update/updated.cambra @@ -0,0 +1,11 @@ +entries := "" + +sign_reqs, sign_resps = http_serve("{PORT}", "POST", "/sign") +peek_reqs, peek_resps = http_serve("{PORT}", "GET", "/peek") + +for entry in sign_reqs: + entries := entries + "- " + entry + "\n" + sign_resps << entries + +for req in peek_reqs: + peek_resps << "peek\n" diff --git a/tests/programs/main.rs b/tests/programs/main.rs index 9dcfec549..afd8694ef 100644 --- a/tests/programs/main.rs +++ b/tests/programs/main.rs @@ -26,6 +26,7 @@ mod http_counter; mod http_greeter; mod inner_join; mod ledger_balance; +mod live_update; mod nonneg_inventory; mod prefix_lines; mod reachability;