diff --git a/BENCHMARK.md b/BENCHMARK.md index d959a6a92..c34eb9d2e 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -788,6 +788,91 @@ target). Detail: `docs/reviews/2026-06-17/4FEATURE-VERIFIED.md`. Moon insert **7.5× (x86) / 14.9× (ARM)** faster; RediSearch search ~16× higher QPS at higher recall (the SQ8/TQ-at-384d recall trade-off). Unchanged from §10.5 — no v3-1/v3-2 regression. +(Note: the search-QPS deficit above predates the vector-search optimization branch (PR #214, +insert 24–38×, matched-recall gap 16×→~1.3×) and the HQ-1 exact-rerank sidecar — see §10.7 +below for post-optimization numbers vs Qdrant.) + +### 10.7 2026-07-07 vs Qdrant (OrbStack Linux VM, Docker Qdrant, same box) + +50K × 384d clustered gaussian (unit-normalized), COSINE, KNN10, 500 queries with exact +ground truth, redis-py / qdrant-client (REST), 8-thread QPS. Both engines at default HNSW +params; recall@10 is matched (≥0.999 both) so QPS compares at equal quality. Branch +`feat/graph-engine-wave2` HEAD. + +| metric | Moon | Qdrant | ratio | +|--------|:----:|:------:|:-----:| +| Ingest rate | **65,695 vec/s** (searchable immediately, brute tier) | 7,350 vec/s accepted / 6,399 vec/s to index-green | **8.9×** | +| Time to HNSW-quality serving | 28.7 s (`FT.COMPACT`, incl. 6 s grace) | **7.8 s** (optimizer green) | 0.27× | +| Search QPS (HNSW, 8 threads) | **3,092** | 1,223 | **2.53×** | +| Search p50 / p99 | **2.60 / 4.39 ms** | 6.05 / 14.66 ms | 2.3× / 3.3× | +| Recall@10 | 0.9992 | 0.9998 | parity | + +Moon wins ingest 8.9× and matched-recall search 2.5×; Qdrant reaches HNSW-tier serving +faster after bulk load (Moon serves immediately from the brute tier during that window — +recall 0.759 there, the documented TQ-at-384d quantized-brute trade-off; low brute QPS at +50K is expected O(N) scan). Caveats: qdrant-client REST transport (gRPC would improve +Qdrant's client-side latency somewhat); shared-host VM — ratios are the signal, absolutes +are indicative. + +### 10.8 2026-07-07 vs Qdrant — GCE validation (dedicated instances, x86 + ARM) + +Same workload and harness as §10.7, re-run on dedicated GCE instances to validate the +same-box VM ratios on real cloud hardware: **c3-standard-8** (Xeon Platinum 8481C, x86_64) +and **t2a-standard-8** (Neoverse-N1, aarch64), us-central1-a. Fresh native build of branch +HEAD `852d53a` on each instance; Docker Qdrant on the same box. + +| metric | x86 Moon | x86 Qdrant | ARM Moon | ARM Qdrant | +|--------|:--------:|:----------:|:--------:|:----------:| +| Ingest rate (vec/s) | **34,121** (searchable immediately) | 3,383 accepted / 3,167 to green | **26,034** (searchable immediately) | 2,383 accepted / 2,273 to green | +| Time to HNSW-tier serving | 27.2 s | **15.8 s** | 47.6 s | **22.0 s** | +| Search QPS (HNSW, 8 threads) | **2,097** | 613 | **1,422** | 531 | +| Search p50 / p99 (ms) | **3.72 / 5.61** | 11.98 / 23.88 | **5.50 / 7.98** | 13.75 / 26.11 | +| Recall@10 | 0.9992 | 1.0000 | 0.9992 | 0.9998 | + +Moon ingests **10.1× (x86) / 10.9× (ARM)** faster and serves matched-recall search +**3.4× (x86) / 2.7× (ARM)** faster — confirming §10.7's VM reading (2.53×) on dedicated +cloud hardware. Qdrant keeps its time-to-index-green edge after bulk load (Moon serves +from the brute tier in that window). Recall is computed against exact ground truth +(full-precision brute-force top-10 over all 50K vectors per query); both engines ≥0.999, +within 0.0008 of each other, and Moon's 0.9992 reproduces bit-identically across all +three environments (VM, GCE x86, GCE ARM). + +### 10.9 2026-07-08 time-to-index-green: parallel HNSW build + insert-path trigger — Moon now beats Qdrant (GCE, x86 + ARM) + +§10.8's one losing metric fixed (commit `061c73cb`, same instances/harness). Instrumentation +showed 99.3% of FT.COMPACT wall was a single-threaded HNSW insert loop, compounded by a +Linux affinity trap (threads spawned from core-pinned shard threads inherit the single-core +mask — `available_parallelism()` returned 1, silently serializing the "parallel" path AND +sizing the background-compactor pool to one worker) and by the auto-compact trigger living +only on the search path (a pure bulk load never compacted until the first FT.COMPACT). +Fixes: shared-graph concurrent HNSW builder (per-node locks + connectivity repair, builds +≥10K vectors, ~88% scaling efficiency), affinity-independent `system_parallelism()` with +explicit worker re-pinning, and an HSET-path compaction trigger (builds start + install +during ingest). + +**Load → HNSW-tier serving** (50K × 384d bulk load, then immediate FT.COMPACT, no +measurement traffic): + +| | x86 before | **x86 after** | x86 Qdrant | ARM before | **ARM after** | ARM Qdrant | +|---|:---:|:---:|:---:|:---:|:---:|:---:| +| load → green | ~22.7 s | **9.9 s** | 15.7 s | ~43.5 s | **9.5 s** | 22.2 s | + +Moon now reaches HNSW-tier serving **1.6× (x86) / 2.3× (ARM) faster than Qdrant** — every +§10.8 metric is now a Moon win. Full-workload re-run on the same boxes: + +| metric | x86 Moon | x86 Qdrant | ARM Moon | ARM Qdrant | +|--------|:--------:|:----------:|:--------:|:----------:| +| Ingest rate (vec/s) | **19,792** | 3,408 accepted | **24,519** | 2,357 accepted | +| Search QPS (HNSW, 8 threads) | **2,266** | 626 | **1,480** | 532 | +| Search p50 / p99 (ms) | **3.55 / 4.04** | 11.64 / 24.12 | **5.37 / 6.20** | 13.64 / 26.74 | +| Recall@10 | 0.9982 | 1.0000 | 0.9980 | 0.9998 | + +Honest trade-offs: (1) ingest rate drops vs §10.8 (34K→20K x86) because HNSW builds now +run concurrently with ingest — the same trade Qdrant makes (its 3.4K/s accept rate IS its +indexing); Moon still ingests 5.8–10.4× faster. (2) recall@10 dips 0.9992 → 0.998x — +the index now serves from 3 segments instead of one (multi-segment beam truncation), a +~0.001 recall cost for the 2.3–4.6× faster time-to-green; still within 0.002 of Qdrant. + --- ## 11. Graph Engine @@ -861,6 +946,131 @@ label scan** — a property index for inline-equality is a legitimate future opt v3-2 scope. Moon native builds 23–26× faster and native 1-hop edges out FalkorDB Cypher. Detail: `docs/reviews/2026-06-17/4FEATURE-VERIFIED.md`. +### 11.6 2026-07-07 Graph wave-2 criterion microbenchmarks (OrbStack Linux VM, aarch64) + +Wave-2 engine (`feat/graph-engine-wave2`, PR #237): frozen-tier copy-up writes, IndexScan +ranges, row-BFS multi-segment gate, write-side plan cache, Cypher aggregations, +OPTIONAL MATCH/WITH. `-C target-cpu=native`, fat-LTO bench profile, criterion. +VM cores are shared — treat absolutes as indicative, relatives as solid. + +| Benchmark | Median | Notes | +|-----------|:------:|-------| +| 1-hop neighbor, CSR (frozen) | **1.07 ns** | vs memgraph (mutable) 113.8 ns — frozen tier ~106× | +| 2-hop BFS, CSR 1K / 10K | 1.01 / 1.10 µs | vs memgraph 3.47 / 3.64 µs (~3.3×) | +| Edge insert (memgraph) | 206 ns | | +| GRAPH.ADDNODE / ADDEDGE dispatch | 223 / 204 ns | ~4.5–4.9M ops/s per-shard ceiling | +| GRAPH.NEIGHBORS dispatch | 429 ns | | +| CSR freeze, 64K edges | 19.5 ms | per-dirty-graph cost of the checkpoint graph snapshot (P0 fix) | +| Cosine similarity 384d / 768d | 31 / 57 ns SIMD | vs scalar 247 / 522 ns (~8–9×, NEON) | +| Row-BFS frozen 1-segment, 10K depth-3 | **658 µs** | vs sequential memgraph BFS 9.44 ms (~14×) | +| Row-BFS frozen 2-segment | 1.65 ms | W2-6 multi-segment gate ≈ 2.5× single-segment | +| Reader BFS mixed-tier / frozen-2seg | 1.83 / 3.40 ms | | +| `ParallelBfs` memgraph 10K depth-3 | 11.8 ms | **slower than sequential 9.44 ms — see note** | + +**`ParallelBfs` note:** the memgraph parallel path is a structural net loss and has **zero +production callers** (the query path uses `BoundedBfs`; both share the row-BFS frozen fast +path, which is where parallelism actually pays). Its neighbor collection stays sequential +(`SegmentMergeReader` borrows `!Send` MemGraph), so it parallelizes only visited-set +filtering while paying per-level neighbor-list clones, a full FxHashSet→DashSet visited +rebuild, and raw `thread::scope` spawns per 128-node morsel (~76 spawns/level at a 9.7K +frontier). Candidate cleanup: retire it or fold into `BoundedBfs`. + +### 11.7 2026-07-07 wave-2 vs FalkorDB (OrbStack Linux VM, Docker FalkorDB) + +`scripts/bench-graph-compare.sh --nodes 5000` (5K nodes, 3K edges), wave-2 engine. +⚠ Sequential redis-cli harness (one process per command) — per-op cost is dominated by +the ~1 ms redis-cli fork on BOTH sides, so this measures single-client latency deltas, +not server throughput; the §11.5 8-thread persistent-connection GCloud run remains the +throughput reference. Ratios: + +| Operation | Moon | FalkorDB | Ratio | +|-----------|:----:|:--------:|:-----:| +| Node insert | 880/s | 716/s | **1.2×** | +| Edge insert | 996/s | 580/s | **1.7×** | +| 1-hop query | 938/s | 684/s | **1.3×** | +| 2-hop query | 1,000/s | 724/s | **1.3×** | +| Cypher pattern match | 862/s | 757/s | **1.1×** | + +Moon leads every row in this harness — including Cypher, where §11.5 (pre-wave-2, +concurrent harness) trailed ~4×. The two harnesses are not directly comparable +(fork-bound single client compresses server-side deltas); a fresh 8-thread GCloud run +is the right follow-up before claiming the Cypher gap is closed. + +### 11.7b 2026-07-07 Cypher-2× wave (P1 mutable property index, P2 result cache, P3 text predicates) + +Follow-up to §11.7/§11.8: profiling showed 82.5% of shard CPU in `index_scan_keys`'s +mutable-tier linear scan. Three features landed (commits `8d5794b9`, `9b38f56c`+`1abc4998`, +`9d634607`): + +- **P1 — mutable-tier property index**: point queries stop scanning every memgraph node. + Quiet-host measurement (OrbStack, 8-thread, 5K nodes): 25,506 → **29,014 qps**, p50 + 0.30 → **0.22 ms** (−27%), shard CPU −~70% (`index_scan_keys` GONE from perf profile; + top symbol drops to 3.9% kernel TCP). Same-box FalkorDB ratio 2.44× → **2.78×**. + The win scales with graph size: the bench scans only 5K nodes; at 1M the old path is + ~200× more per-query while the index probe stays O(1). +- **P2 — Cypher result cache** (write-gen invalidated, pre-encoded RESP bytes, TinyLFU + doorkeeper): interleaved A/B vs P1-only (noisy shared host, alternating runs): + cache-hostile cycling **−2.4%** (was −21% before the doorkeeper — admission on second + sighting removed the serialize+insert+O(n)-evict miss cost), hot-key repeat **+1.9%**. + Real payoff is expensive reads (a 24 ms full-graph aggregation becomes a byte-copy), + not client-bound point queries. +- **P3 — text predicates via FTS reuse**: `CONTAINS` / `STARTS WITH` / `ENDS WITH` (new + syntax) + `=~` now prune frozen segments through a `SegmentTextIndex` (presence + bitmap — deliberately NOT token-postings: `CONTAINS 'rust'` must match `"trusted"`, + so tokenized pruning would be unsound; residual Filter stays authoritative). BM25 + scoring path implemented + tested, Cypher surface syntax deferred. + +### 11.8 2026-07-07 wave-2 vs FalkorDB — 8-thread GCloud harness: **Cypher gap CLOSED** + +Same harness as §11.5 (graph phase of `gce-4feature-bench.sh`: 5K nodes, 15K edges, +redis-py, 8 threads, 6s per op class), wave-2 engine @ 1bbaec61, same-box FalkorDB +(Docker). ⚠ Instance is **e2-standard-16** (2.2 GHz shared-core Xeon; c2d capacity +exhausted in-zone), much weaker than §11.5's machines — cross-run absolutes are NOT +comparable, but the same-box Moon:FalkorDB ratio is the valid metric. + +| metric | Moon (wave-2) | FalkorDB | ratio | §11.5 ratio (pre-wave-2) | +|--------|:-------------:|:--------:|:-----:|:------------------------:| +| cypher_1hop qps (p50) | 3,489 (1.77 ms) | 3,879 (2.02 ms) | **0.90×** | 0.25× | +| cypher_2hop qps (p50) | **4,518** (0.89 ms) | 3,701 (2.11 ms) | **1.22×** | 0.26× | +| build ops/s | **10,321** | 535 | **19×** | 26× | +| cypher_match_rows | 5 | 5 | correct parity | 4 = 4 | +| native_neighbors qps | 3,185 | — | — | — | + +The June deficit — FalkorDB's property index vs Moon's filtered label scan — is gone +at this scale: Moon Cypher point queries improved ~3–4× relative to FalkorDB on the +same box (wave-2 write-side plan cache + IndexScan + executor work). Moon now WINS +2-hop at better p50 and ties 1-hop within 10% on the weakest instance class; on equal +§11.5-class hardware the 1-hop tie likely flips too (**confirmed in §11.9** — dedicated +cores flip 1-hop to a 2.3–2.7× Moon win). Caveat: Moon's p99 (18.9 ms 1-hop) trails +FalkorDB's (3.1 ms) on this shared-core instance — §11.9 shows this was a shared-core +artifact (dedicated-core p99 beats FalkorDB). Interesting inversion: Moon Cypher 1-hop (3,489 qps) now beats its own +native GRAPH.NEIGHBORS (3,185 qps) under concurrency — the plan cache amortizes +parsing to near-zero. + +### 11.9 2026-07-07 Cypher-2× wave vs FalkorDB — GCE dedicated-core validation (x86 + ARM) + +Supersedes §11.8's shared-core e2 caveat. Same 8-thread 1-hop Cypher point-query workload +(5K nodes / 15K edges, seed(7), 20 s measure, redis-py persistent connections), run on +dedicated GCE instances — **c3-standard-8** (Xeon Platinum 8481C) and **t2a-standard-8** +(Neoverse-N1) — with Docker FalkorDB on the same box, 2 interleaved reps each. Fresh +native build of branch HEAD `852d53a` (post P1 property index / P2 result cache / P3). + +| metric | x86 Moon | x86 FalkorDB | ratio | ARM Moon | ARM FalkorDB | ratio | +|--------|:--------:|:------------:|:-----:|:--------:|:------------:|:-----:| +| cypher_1hop qps (rep1 / rep2) | **11,641 / 11,620** | 4,988 / 4,955 | **2.34×** | **10,938 / 10,867** | 4,028 / 4,135 | **2.67×** | +| p50 (ms) | **0.54** | 1.46 | 2.7× | **0.61** | 1.81 | 2.9× | +| p90 (ms) | **1.36** | 2.30 | — | **1.33** | 2.83 | — | +| p99 (ms) | **2.48** | 3.33 | — | **2.26** | 4.31 | — | + +Moon **wins 1-hop Cypher outright on both architectures** — 2.3–2.7× the QPS at ~2.7–2.9× +better p50 — and rep-to-rep spread is <0.2% (dedicated cores). The §11.8 p99-tail concern +(18.9 ms on the shared-core e2) is confirmed as a shared-core artifact: on dedicated cores +Moon's p99 *beats* FalkorDB's on both arches. Ratios also validate the same-box VM readings +(§11.7b 2.78×, final-HEAD 2.55×). FalkorDB build rate for context: 1,009 ops/s (x86) / +605 ops/s (ARM) via batched UNWIND — Moon's build side used the sequential single-client +native API in this harness, so build is not compared here (see §11.5 for the 23–26× +concurrent-build comparison). + --- ## 12. Full-Text Search diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b7aff05..2c7d5a009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +<<<<<<< HEAD +### Added — parallel HNSW build + insert-path compaction trigger: time-to-index-green 11× (PR #237) + +- **`src/vector/hnsw/parallel_build.rs`** (new): concurrent HNSW + construction into one shared graph — per-node `parking_lot::Mutex` + adjacency (copy-under-lock, distance math unlocked; single lock held at a + time, deadlock-free by construction), entry point under a `RwLock`, + levels pre-generated from the same seeded LCG as the sequential builder, + sequential 1K warmup, then dynamic fan-out over an atomic cursor. + Finalize adds a connectivity repair pass (concurrent back-link pruning + orphaned ~0.2% of nodes = permanent recall loss; BFS + force-link makes + every node reachable, unit-asserted) and the exact same BFS-reorder → + `HnswGraph` path as the sequential builder — drop-in for search, + persistence, and GraphUnion merge. `compact()` routes builds ≥ 10K + vectors here; smaller segments keep the bitwise-deterministic + single-threaded builder. Measured (50K × 384d, 6-core Linux VM): + `FT.COMPACT` wall 30.2s → **2.71s**; a 24K segment build 14.1s → + **2.65s** (~88% scaling efficiency). +- **Affinity-mask trap fixed (3 sites)**: shard threads are core-pinned and + every thread spawned from one inherits the SINGLE-core mask on Linux, so + `std::thread::available_parallelism()` returned 1 — the "parallel" build + ran at exactly sequential speed and the background-compactor pool sized + itself to one worker. New `shard::numa::system_parallelism()` (sysfs + online-CPU count, affinity-independent, cached) now sizes both; parallel + build workers and pool workers explicitly re-pin round-robin across the + machine (`pin_worker_to_core`), pool workers from the last core downward + so small builds stop time-slicing shard 0's core. +- **`src/shard/spsc_handler.rs`**: HSET auto-index hook now calls + `try_compact()` after appending a vector — a pure bulk load (no + FT.SEARCH traffic) previously left everything in the brute-force mutable + tier until the autovacuum backstop's 30s tick, so the whole HNSW build + landed on the first explicit `FT.COMPACT`. Builds now start and install + DURING ingest (in-flight guard unchanged; respects + `FT.CONFIG AUTOCOMPACT OFF`). Red/green: new + `test_insert_path_triggers_background_compact_without_search`. +- Net effect on the §10.8 losing metric (bulk load 50K × 384d → HNSW-tier + serving, VM): ~30s of post-ingest compaction becomes ~0 (already + compacted by measure time) — Moon's time-to-green now beats Qdrant's + optimizer on the same box (GCE re-validation pending). Multi-segment + recall@10 0.9986 vs 0.9992 single-segment (−0.0006, within the ≥0.99 + gate); multi-shard verified (`--shards 4`: per-shard triggers, 9 + segments, FT.COMPACT 1.57s). + ### Docs — durability write-path benchmark results (PR #TBD) - `BENCHMARK.md` §7.3: new section recording the 2026-07-08 durability @@ -505,7 +548,122 @@ path (below), or replay them on a pre-freeze build first and re-persist. HNSW bridge (heap-owned; mmap-backed sections count 0 per the `RawF16Store` precedent), keeping the elastic memory budget honest. -### Changed — consolidated dependency bumps (PR #TBD) +### Changed — graph engine wave 2: durability, Cypher coverage, hardening (PR #TBD) + +- **Durability (P0, found by GCP production-hardening soak):** graph data now + survives kill -9 under the DEFAULT disk-offload (WAL v3) configuration. + Two stacked pre-existing bugs erased graphs on crash: (A) v3 recovery + collected graph WAL commands into a throwaway replay engine — graph replay + was a complete no-op in v3 mode (a dedicated `replay_graph_wal_v3` boot + pass now applies them); (B) the checkpoint advanced the WAL replay floor + and recycled segments without ever snapshotting the graph store — + `save_graph_store` ran only on graceful shutdown and never persisted the + mutable tier (checkpoint finalize now freezes each graph's write buffer, + persists segments + a `snapshot_lsn` replay floor, and ABORTS the + checkpoint if the graph snapshot fails, keeping the old floor). The prior + crash suite ran exclusively with `--disk-offload disable`; new G4/G5 + scenarios cover the default config with and without a checkpoint crossing. +- **Durability:** stable external node/edge ids across WAL replay (handles + handed to clients before a crash resolve to the same rows after recovery); + Cypher `SET` property/label writes now emit WAL records + (`GRAPH.SETPROP`/`GRAPH.SETLABEL` — previously a documented durability gap: + SET mutations silently vanished on kill -9); new kill-9 crash-recovery + suite (`tests/crash_recovery_graph_durability.rs`: mutable tier, frozen + 64K-edge tier rebuilt through replay-time freeze, double-crash idempotence). +- **Cypher coverage:** aggregations `count`/`sum`/`avg`/`min`/`max`/`collect` + with implicit grouping, `DISTINCT`, and count-over-zero-rows semantics; + `OPTIONAL MATCH` null-pads unmatched expansions (previously compiled + silently to inner MATCH; unsupported shapes now reject loudly); + `WITH` rebinds the pipeline mid-query (aggregate + `WHERE`-as-HAVING, + `ORDER BY`/`SKIP`/`LIMIT` between WITH and RETURN; previously every clause + after WITH ran on an empty row stream). `WITH *` rejects loudly. +- **Copy-up writes:** `SET`/`DELETE`/`MERGE` on frozen rows copy the row up + into the write buffer instead of silently missing the frozen tier. +- **Query performance:** IndexScan range predicates (`WHERE n.p > x` prunes + via per-segment B-tree-ish numeric index, superset semantics + residual + filter); write-side plan cache (repeated write shapes skip parse+compile); + row-BFS fast path engages on multi-segment fully-frozen graphs (~4× vs + reader fallback, criterion-pinned); `Value::String` holds `Bytes` for a + zero-copy reply path; HYB-02/HYB-04 use the HNSW bridge with off-thread + bridge builds. +- **Query performance — mutable-tier property index (task #31):** `MATCH + (a:N {id: X})` on the write buffer used to degrade to a full + `O(live_node_count)` linear scan even though `PhysicalOp::IndexScan` was + already the plan (profiling on the 5K-node/15K-edge Cypher point-query + bench put this scan at 82.5% of shard CPU — the mutable tier never + freezes below `edge_threshold`, default 64,000). A new incrementally + maintained `MutablePropertyIndex` (`src/graph/index.rs`, mirrors the + frozen tier's `SegmentPropertyIndexes` numeric-BTree/string-hash split, + keyed by `NodeKey`) turns the mutable-tail probe into an O(log N + + |result|) index seed; `index_scan_keys`'s residual label/MVCC/property + checks are unchanged (SUPERSET contract preserved, no planner changes). + `MemGraph::set_node_property`/`remove_node_property`/`undelete_node` are + now the single source of truth for node-property mutation, replacing 5 + previously hand-rolled call sites (Cypher `SET`, MERGE ON CREATE/MATCH + SET, TXN.ABORT undo, WAL replay) that could silently let the index drift + from live state. Closes a TXN.ABORT `UndeleteNode` gap found during + design review: undoing a `DELETE` inside a transaction now re-indexes the + restored node instead of leaving it live but permanently unreachable by + index probes. +- **Query performance — Cypher result cache (task #32):** read-only + `GRAPH.QUERY`/`GRAPH.RO_QUERY` now cache the fully-encoded RESP reply + bytes for repeated identical queries (same raw Cypher text + args), + keyed by `(query_hash, args_hash)` and served via the existing + `Frame::PreSerialized` passthrough — no new protocol variant needed, and + both RESP2 and RESP3 encodings are cached in separate slots so a + protocol-version switch is a clean miss rather than a stale re-encode. + Invalidation is a per-graph monotonic `write_gen: u64` bumped by + `NamedGraph::touch()` at every real mutation site (plain `GRAPH.ADDNODE`/ + `GRAPH.ADDEDGE`, the Cypher write-plan mutation loop, `TS.*` temporal + invalidation, and TXN.ABORT rollback of graph undo-ops) — deliberately + NOT at `freeze_and_compact`, which reshapes storage without changing + query-visible content. The write_gen is captured before executing a + read and the encoded reply is only cached if it is still unchanged + after — a miss is always safe (SUPERSET semantics), so a benign race + just skips caching rather than serving stale data. Decayed queries and + errored/timed-out results are never cached. `ResultCache` is a bounded + per-graph LRU (256 entries / 4MiB default, `src/graph/cypher/ + result_cache.rs`) whose resident bytes are folded into + `GraphStore::resident_bytes()`; dropping a graph (`GRAPH.DELETE`) drops + its cache with it. The cache is only consulted on the connection-local + dispatch path where the negotiated protocol version is reliably known + (`dispatch_graph_read`, threaded as `Option`); the cross-shard + `GraphCommand` hop and `graph_query_or_write`'s internal read-execute + calls pass `None` and bypass the cache entirely rather than risk an + unverified protocol version. +- **Query performance — FTS reuse for Cypher text predicates (task #33):** + first-class `CONTAINS`/`STARTS WITH`/`ENDS WITH` Cypher operators (pure + syntax sugar over the existing `=~` byte-level checks — `src/graph/ + cypher/lexer.rs`, `ast.rs`, `parser/expr.rs`, `executor/eval.rs`), plus a + new per-frozen-segment `SegmentTextIndex` (`src/graph/text_index.rs`, + lazy `OnceLock` + `resident_bytes` accounting, same pattern as + `SegmentPropertyIndexes`/`hnsw_bridge`) that accelerates `CONTAINS`/ + `STARTS WITH`/`ENDS WITH`/`=~` conjuncts in `WHERE` (`planner.rs`'s new + `extract_text_conjuncts`, mirroring W2-3's `extract_range_conjuncts`). + Reuses `crate::text::posting::PostingStore`/`crate::text::bm25:: + {FieldStats, bm25_score}`/`crate::text::term_dict::TermDictionary` + verbatim (already ID-space-agnostic, zero-fork). **Correctness note:** + the index prunes on PROPERTY PRESENCE only, not token identity — a + tokenized/stemmed/case-folded posting lookup cannot safely decide + substring/prefix/suffix containment (e.g. `CONTAINS 'rust'` must also + match `"trusted"`, which tokenizes to a different term entirely), so + `candidate_rows` returns every row whose target property is a + String/Bytes at all, and the existing residual `Filter` remains the sole + decider (SUPERSET contract, same as `prop_eq`/`prop_range`). The + MUTABLE tier gets no acceleration (falls back to an exact scan of the + write buffer, pre-approved scope decision — mirrors the pre-task-#31 + numeric story); a `GraphUnion`-merged segment does not remap posting row + ids and simply rebuilds its text index lazily on first use post-merge. + `SegmentTextIndex::bm25_score_for` is a tested-but-unwired internal BM25 + relevance-scoring hook (no Cypher `ORDER BY` grammar was specified for + it) — proves the `bm25_score` reuse end-to-end; surface syntax is a + documented follow-up. +- **Operability:** traversal timeout is configurable — `--graph-timeout-ms` + server default plus per-query `GRAPH.QUERY ... TIMEOUT ` (RedisGraph + parity, 0 = unlimited). +- **Dead code:** cross-shard traverse scaffolding deleted (532 lines, no + sender existed); the single-shard-per-graph sharding model is now + documented in `src/graph/mod.rs`. - Cargo: `ringbuf` 0.4.8 → 0.5.0 and `metrics-exporter-prometheus` 0.16.2 → 0.18.3 (semver-major; both compile and test green with no code changes — diff --git a/benches/graph_traversal.rs b/benches/graph_traversal.rs index e390e3c5a..ed947ca2a 100644 --- a/benches/graph_traversal.rs +++ b/benches/graph_traversal.rs @@ -139,5 +139,119 @@ fn bench_parallel_vs_sequential_bfs(c: &mut Criterion) { group.finish(); } -criterion_group!(graph_traversal, bench_parallel_vs_sequential_bfs); +// --------------------------------------------------------------------------- +// Frozen-tier BFS benchmarks (W2-10) +// +// The CSR row-space fast path (`row_bfs`) only engages on a fully-frozen +// graph — the mutable-only fixtures above never exercise it. These fixtures +// freeze the SAME 10K/50 graph so the three production shapes are directly +// comparable: +// frozen_single — one CSR segment: full row-BFS (parallel levels + Beamer) +// frozen_multi2 — the graph frozen twice (identical clone segments, the +// W2-2 copy-up aftermath shape): W2-6 multi-segment row +// path with worst-case key-level boundary sync (every node +// resident in both segments) +// mixed_tier — same segment + a non-empty mutable tail: the gate +// declines and BFS falls back to the SegmentMergeReader +// path (what row-BFS saves) +// --------------------------------------------------------------------------- + +fn bench_frozen_tier_bfs(c: &mut Criterion) { + use moon::graph::csr::CsrSegment; + use slotmap::Key; + + let mut group = c.benchmark_group("frozen_bfs"); + + const N: usize = 10_000; + const DEGREE: usize = 50; + + // Freeze the standard fixture once → single segment. + let (mut g, nodes) = build_memgraph(N, DEGREE); + let seed = nodes[500]; + let seg_old = Arc::new(CsrStorage::from( + CsrSegment::from_frozen(g.freeze().expect("freeze"), 3).expect("csr"), + )); + + // Re-materialize every node at its ORIGINAL key and re-add the same LCG + // edges, then freeze again → an identical clone segment (the shape a + // W2-2 copy-up + re-freeze leaves behind). Every node is resident in + // BOTH segments — worst-case boundary sync for the multi-segment path. + g.thaw(); + for &nk in &nodes { + g.add_node_with_id(nk.data().as_ffi(), smallvec![0], empty_props(), None, 4); + } + let mut rng_state: u32 = 42; + for i in 0..N { + for _ in 0..DEGREE { + rng_state = rng_state.wrapping_mul(1664525).wrapping_add(1013904223); + let target = (rng_state as usize) % N; + if target == i { + continue; + } + let _ = g.add_edge(nodes[i], nodes[target], 1, 1.0, None, 5); + } + } + let seg_new = Arc::new(CsrStorage::from( + CsrSegment::from_frozen(g.freeze().expect("freeze"), 6).expect("csr"), + )); + + let single: Vec> = vec![seg_old.clone()]; + let multi: Vec> = vec![seg_new, seg_old.clone()]; + + // Mixed tier: one live node in the write buffer flips the gate off and + // forces the reader fallback over the same frozen data. + let mut tail = MemGraph::new(usize::MAX >> 1); + let _ = tail.add_node(smallvec![0], empty_props(), None, 7); + + let run = |mg: Option<&MemGraph>, segs: &[Arc]| { + let reader = SegmentMergeReader::new(mg, segs, Direction::Outgoing, u64::MAX - 1, None); + BoundedBfs::new(3).execute(&reader, seed) + }; + + // Correctness: all three shapes must visit the same node set (the clone + // segment adds no new reachability). + { + let key_set = |r: &moon::graph::traversal::BfsResult| { + let mut v: Vec = r.visited.iter().map(|e| e.0.data().as_ffi()).collect(); + v.sort_unstable(); + v + }; + let s = run(None, &single).expect("single ok"); + let m = run(None, &multi).expect("multi ok"); + let x = run(Some(&tail), &single).expect("mixed ok"); + assert_eq!(key_set(&s), key_set(&m), "multi must match single"); + assert_eq!(key_set(&s), key_set(&x), "reader fallback must match"); + eprintln!( + "frozen BFS precheck: 10K nodes, degree 50, depth 3 -> {} nodes visited", + s.visited.len() + ); + } + + group.bench_function("row_bfs_frozen_single_10k_depth3", |b| { + b.iter(|| black_box(run(None, black_box(&single)))) + }); + + group.bench_function("row_bfs_frozen_multi2_10k_depth3", |b| { + b.iter(|| black_box(run(None, black_box(&multi)))) + }); + + group.bench_function("reader_bfs_mixed_tier_10k_depth3", |b| { + b.iter(|| black_box(run(Some(black_box(&tail)), black_box(&single)))) + }); + + // Apples-to-apples W2-6 baseline: the reader path over the SAME two + // segments (multi2 doubles the edge probes vs `single`, so only this + // pairing isolates the multi-segment row path's win). + group.bench_function("reader_bfs_frozen_multi2_10k_depth3", |b| { + b.iter(|| black_box(run(Some(black_box(&tail)), black_box(&multi)))) + }); + + group.finish(); +} + +criterion_group!( + graph_traversal, + bench_parallel_vs_sequential_bfs, + bench_frozen_tier_bfs +); criterion_main!(graph_traversal); diff --git a/src/command/graph/graph_read.rs b/src/command/graph/graph_read.rs index ebb92bc28..84793ba71 100644 --- a/src/command/graph/graph_read.rs +++ b/src/command/graph/graph_read.rs @@ -77,6 +77,118 @@ fn parse_decay(args: &[Frame]) -> Result` argument (RedisGraph parity). +/// +/// `TIMEOUT 0` disables the timeout for this query. Strict validation like +/// `parse_decay` (new surface ⇒ malformed input is an error, not a silent +/// no-op): a dangling keyword or non-integer value is rejected. Returns +/// `Ok(None)` when the keyword is absent. +pub(super) fn parse_timeout_ms(args: &[Frame]) -> Result, &'static str> { + for i in 0..args.len() { + if let Frame::BulkString(ref bs) = args[i] { + if bs.eq_ignore_ascii_case(b"TIMEOUT") { + let Some(Frame::BulkString(val)) = args.get(i + 1) else { + return Err("ERR TIMEOUT requires a value in milliseconds"); + }; + let parsed = std::str::from_utf8(val) + .ok() + .and_then(|s| s.trim().parse::().ok()); + return match parsed { + Some(ms) => Ok(Some(ms)), + None => Err("ERR TIMEOUT must be a non-negative integer (milliseconds)"), + }; + } + } + } + Ok(None) +} + +/// Build the traversal guard for one query: per-query `TIMEOUT ` override +/// if present (0 = unlimited), else the configured process default +/// (`--graph-timeout-ms`, 30s unless overridden). +fn query_guard( + args: &[Frame], + snapshot_lsn: u64, +) -> Result { + use crate::graph::traversal_guard::TraversalGuard; + Ok(match parse_timeout_ms(args)? { + Some(0) => TraversalGuard::new(snapshot_lsn, std::time::Duration::MAX), + Some(ms) => TraversalGuard::new(snapshot_lsn, std::time::Duration::from_millis(ms)), + None => TraversalGuard::with_default_timeout(snapshot_lsn), + }) +} + +/// Cheap pre-check for the Cypher result cache (Task #32): does this +/// GRAPH.QUERY carry a `--decay` flag? Decay queries are wall-clock +/// dependent (`TemporalDecayScorer::now` captures real time at query start, +/// not derived from graph state) and must NEVER be cached regardless of +/// `write_gen` -- two decay queries at different real times against an +/// unchanged graph legitimately score/order differently. Byte-scan only +/// (not a full `parse_decay`) so a miss on this check costs nothing on the +/// hot path; `parse_decay`'s stricter validation still runs later in +/// `run_read_query` regardless of this pre-check's answer. +fn has_decay_flag(args: &[Frame]) -> bool { + args.iter() + .any(|f| matches!(f, Frame::BulkString(b) if b.as_ref() == b"--decay")) +} + +/// Hash the "remaining args" (everything after the graph name and Cypher +/// text -- `--params`, `VALID_AT`, `TIMEOUT`, `--decay`, `--time-weight`) +/// for the Cypher result-cache key (Task #32). Allocation-free: each arg's +/// logical byte content is hashed independently via `xxh64` and folded, +/// rather than concatenated into one buffer first. Order-sensitive (a +/// differently-ordered but semantically-identical arg list gets a different +/// key) -- a harmless false split of the key space, never a correctness +/// issue, since a miss just re-executes and re-populates. +fn hash_query_args(args: &[Frame]) -> u64 { + let mut acc: u64 = 0; + for frame in args { + acc = acc.rotate_left(13) ^ hash_frame_bytes(frame); + } + acc +} + +/// `xxh64` of a single `Frame`'s logical byte content, used only for +/// result-cache key derivation -- NOT a wire format. Numeric/boolean +/// variants hash their shortest text representation (stack-buffer +/// `itoa`/`ryu`, no allocation) rather than their binary encoding; hash +/// collisions across variants are harmless because `ResultCacheKey` +/// equality is a plain struct compare, not the hash alone -- a collision +/// only costs a wasted cache miss, never a wrong answer. +fn hash_frame_bytes(frame: &Frame) -> u64 { + match frame { + Frame::BulkString(b) | Frame::SimpleString(b) => cypher::planner::hash_query(b), + Frame::Integer(n) => { + let mut buf = itoa::Buffer::new(); + cypher::planner::hash_query(buf.format(*n).as_bytes()) + } + Frame::Double(f) => { + let mut buf = ryu::Buffer::new(); + cypher::planner::hash_query(buf.format(*f).as_bytes()) + } + Frame::Boolean(b) => { + cypher::planner::hash_query(if *b { b"\x01true" } else { b"\x01false" }) + } + Frame::Null => cypher::planner::hash_query(b"\x01null"), + _ => cypher::planner::hash_query(b"\x01other"), + } +} + +/// Build the Cypher result-cache key (Task #32) for `cypher_bytes` (the raw +/// query text, `args[1]`) and `rest_args` (everything after it, `args[2..]` +/// -- `--params`/`VALID_AT`/`TIMEOUT`/`--decay`/`--time-weight`). Shared by +/// the pre-lookup in `graph_query_readonly` and the population step in +/// `run_read_query` so both sides always compute the identical key. +fn result_cache_key( + cypher_bytes: &[u8], + rest_args: &[Frame], +) -> cypher::result_cache::ResultCacheKey { + cypher::result_cache::ResultCacheKey { + query_hash: cypher::planner::hash_query(cypher_bytes), + args_hash: hash_query_args(rest_args), + } +} + /// Parse `--params ` from GRAPH.QUERY args into executor `Value` map. /// /// Scans args for `--params` keyword followed by a JSON string. The JSON must be @@ -116,7 +228,9 @@ fn json_to_graph_value(v: &serde_json::Value) -> cypher::executor::Value { cypher::executor::Value::Float(n.as_f64().unwrap_or(0.0)) } } - serde_json::Value::String(s) => cypher::executor::Value::String(s.clone()), + serde_json::Value::String(s) => { + cypher::executor::Value::String(Bytes::copy_from_slice(s.as_bytes())) + } serde_json::Value::Array(arr) => { cypher::executor::Value::List(arr.iter().map(json_to_graph_value).collect()) } @@ -237,8 +351,12 @@ pub fn graph_neighbors(store: &GraphStore, args: &[Frame]) -> Frame { let reader = SegmentMergeReader::new(Some(memgraph), csr_segs, direction, lsn, edge_type_filter); - // TraversalGuard enforces bounded epoch hold (30s default timeout). - let guard = crate::graph::traversal_guard::TraversalGuard::with_default_timeout(lsn); + // TraversalGuard enforces bounded epoch hold (per-query TIMEOUT override, + // else the configured `--graph-timeout-ms` default). + let guard = match query_guard(args, lsn) { + Ok(g) => g, + Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), + }; // BFS expansion using SegmentMergeReader for per-node neighbor lookup. let mut visited = std::collections::HashSet::new(); @@ -472,12 +590,22 @@ fn parse_effective( /// execute`, which rebuilds one every call) so plan-cache hits reuse the /// `SlotTable` cached alongside the plan (Fix 2 -- one `String` allocation /// per bound variable, per execution, otherwise). +/// +/// `cache_protocol_version` (Task #32): `Some(v)` enables result-cache +/// POPULATION after a successful execution, encoding the reply for RESP +/// version `v` (2 or 3) into `graph.result_cache`. `None` means this call +/// site is not wired into the result cache at all (e.g. `graph_query_or_ +/// write`'s read branches -- see module docs for the scope boundary): no +/// lookup happens here regardless (a hit is handled entirely by the +/// caller, BEFORE `run_read_query` is invoked at all -- this function only +/// ever runs on a miss), so `None` just skips the population step. fn run_read_query( graph: &crate::graph::store::NamedGraph, args: &[Frame], plan: &cypher::PhysicalPlan, slots: &cypher::executor::SlotTable, auto_params: Vec<(String, cypher::executor::Value)>, + cache_protocol_version: Option, ) -> Frame { let mut params = parse_params(args); for (name, value) in auto_params { @@ -488,14 +616,61 @@ fn run_read_query( Ok(d) => d, Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), }; + let guard = match query_guard(args, 0) { + Ok(g) => g, + Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), + }; let ctx = cypher::executor::ExecutionContext { valid_time_as_of: valid_at, decay, - guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), + guard: Some(guard), ..Default::default() }; + // Race guard (Task #32): capture the write generation BEFORE execution, + // store only if it is still unchanged AFTER -- on this shard thread + // nothing else can mutate `graph` while this synchronous call is + // running, but capturing before/comparing after costs one extra `u64` + // read and keeps the invariant correct-by-construction even if this + // function ever grows a yield point. + let write_gen_before = graph.write_gen; match cypher::executor::execute_with_slots(graph, plan, slots, ¶ms, &ctx) { - Ok(r) => exec_result_to_frame(&r), + Ok(r) => { + if let Some(protocol_version) = cache_protocol_version { + // Never cache decay queries (wall-clock dependent) or a + // result computed against a graph state that mutated + // mid-call (write_gen_before must still hold). + if decay.is_none() + && graph.write_gen == write_gen_before + && !args.is_empty() + && args.len() >= 2 + { + if let Some(cypher_bytes) = extract_bulk(&args[1]) { + let key = result_cache_key(cypher_bytes, &args[2..]); + // Doorkeeper: admit only keys seen before, so a + // scan/cycle workload's one-shot queries never pay + // the serialize+insert+evict miss cost (measured + // −21% qps on cycling without this gate). + if graph.result_cache.lock().should_admit(key) { + let frame = exec_result_to_frame(&r); + let mut buf = bytes::BytesMut::new(); + if protocol_version >= 3 { + crate::protocol::serialize_resp3(&frame, &mut buf); + } else { + crate::protocol::serialize(&frame, &mut buf); + } + graph.result_cache.lock().put( + key, + write_gen_before, + protocol_version, + buf.freeze(), + ); + return frame; + } + } + } + } + exec_result_to_frame(&r) + } Err(e) => { let msg = format!("ERR Cypher execution error: {e}"); Frame::Error(Bytes::from(msg)) @@ -508,15 +683,30 @@ fn run_read_query( /// Normalizes literals into auto-parameters, then executes via the plan /// cache: a cache hit runs with ZERO parse/compile work (the cache holds /// read-only plans only, so a hit is safe to execute directly). -pub fn graph_query(store: &GraphStore, args: &[Frame]) -> Frame { - graph_query_readonly(store, args, false) +/// +/// `protocol_version` (Task #32): `Some(v)` is the caller's negotiated RESP +/// version (2 or 3), threaded through to the Cypher result cache so a hit +/// replays correctly-encoded wire bytes and a miss populates the right +/// slot. `None` disables the result cache entirely for this call -- used by +/// call sites that cannot reliably determine the originating connection's +/// protocol version (e.g. the cross-shard `ShardMessage::GraphCommand` hop); +/// serving a wrong-protocol cached reply would be a real correctness bug +/// (RESP2/RESP3 wire formats differ), so those sites opt out rather than +/// guess. +pub fn graph_query(store: &GraphStore, args: &[Frame], protocol_version: Option) -> Frame { + graph_query_readonly(store, args, false, protocol_version) } /// Shared read-path core for GRAPH.QUERY / GRAPH.RO_QUERY. /// /// `reject_writes`: RO_QUERY refuses write clauses with an explicit error; /// GRAPH.QUERY lets the read-only executor report them (it has no write lock). -fn graph_query_readonly(store: &GraphStore, args: &[Frame], reject_writes: bool) -> Frame { +fn graph_query_readonly( + store: &GraphStore, + args: &[Frame], + reject_writes: bool, + protocol_version: Option, +) -> Frame { if args.len() < 2 { return Frame::Error(Bytes::from_static( b"ERR wrong number of arguments for 'GRAPH.QUERY' command", @@ -538,24 +728,58 @@ fn graph_query_readonly(store: &GraphStore, args: &[Frame], reject_writes: bool) None => return Frame::Error(Bytes::from_static(b"ERR invalid Cypher query")), }; + // Task #32: Cypher result-cache lookup BEFORE any plan-cache/parse work + // -- a hit skips plan lookup, param parsing, guard construction, and + // execution entirely. Decay queries (wall-clock dependent) never + // consult the cache regardless of `write_gen` freshness. + if let Some(pv) = protocol_version { + if !has_decay_flag(&args[2..]) { + let key = result_cache_key(cypher_bytes, &args[2..]); + if let Some(bytes) = graph.result_cache.lock().get(key, graph.write_gen, pv) { + return Frame::PreSerialized(bytes); + } + } + } + // Raw-hash pre-lookup (Fix 2): an EXACT repeat of a query text we've // already compiled hits here without ever calling `parameterize()` (a // full lexer pass + Vec/String allocations) — the raw-hash entry // carries this exact text's auto_params, cached at insert time. let raw_hash = cypher::planner::hash_query(cypher_bytes); if let Some(cached) = graph.plan_cache.lock().get(raw_hash) { - let auto_params = cached.auto_params.as_ref().clone(); - return run_read_query(graph, args, &cached.plan, &cached.slots, auto_params); + // W2-7 caches WRITE plans too — a write hit falls through to the + // parse path, which reports it exactly like an uncached write query. + if cached.read_only { + let auto_params = cached.auto_params.as_ref().clone(); + return run_read_query( + graph, + args, + &cached.plan, + &cached.slots, + auto_params, + protocol_version, + ); + } } let (effective, query_hash, auto_params) = normalize_cypher(cypher_bytes); - // Normalized-hash hit ⇒ read-only plan (PlanCache invariant) ⇒ no parse. - // `parameterize()` above already ran (needed to derive THIS text's - // auto_params and the normalized hash), but parse+compile is skipped. + // Normalized-hash READ-ONLY hit ⇒ no parse. `parameterize()` above + // already ran (needed to derive THIS text's auto_params and the + // normalized hash), but parse+compile is skipped. A cached WRITE plan + // (W2-7) falls through to the parse path instead. let cached = graph.plan_cache.lock().get(query_hash); if let Some(cached) = cached { - return run_read_query(graph, args, &cached.plan, &cached.slots, auto_params); + if cached.read_only { + return run_read_query( + graph, + args, + &cached.plan, + &cached.slots, + auto_params, + protocol_version, + ); + } } let (query, query_hash, auto_params) = @@ -577,19 +801,23 @@ fn graph_query_readonly(store: &GraphStore, args: &[Frame], reject_writes: bool) return Frame::Error(Bytes::from(msg)); } }; - // Cache read-only plans ONLY — a cache hit skips classification, so a - // cached write plan would execute on the read path. Insert under both - // the raw and normalized hash so a later exact repeat of this text hits - // the allocation-free raw-hash path above. + // Cache read-only plans only on this handler — write plans are cached + // (flagged read_only=false) by the write handlers (W2-7), and every hit + // above re-checks the flag. Insert under both the raw and normalized + // hash so a later exact repeat of this text hits the allocation-free + // raw-hash path above. let slots = if query.is_read_only() { - graph - .plan_cache - .lock() - .insert_both(raw_hash, query_hash, plan.clone(), auto_params.clone()) + graph.plan_cache.lock().insert_both( + raw_hash, + query_hash, + plan.clone(), + auto_params.clone(), + true, + ) } else { std::sync::Arc::new(cypher::executor::SlotTable::from_plan(&plan)) }; - run_read_query(graph, args, &plan, &slots, auto_params) + run_read_query(graph, args, &plan, &slots, auto_params, protocol_version) } /// GRAPH.QUERY — write-capable variant. @@ -613,11 +841,38 @@ pub fn graph_query_write(store: &mut GraphStore, args: &[Frame]) -> Frame { None => return Frame::Error(Bytes::from_static(b"ERR invalid Cypher query")), }; - let query = match cypher::parse_cypher(cypher_bytes) { - Ok(q) => q, - Err(e) => { - let msg = format!("ERR Cypher parse error: {e}"); - return Frame::Error(Bytes::from(msg)); + // W2-7: literal-normalize + plan-cache for the write path too. A hit on + // a cached WRITE plan skips parse/compile entirely; per-run literal + // values arrive through the auto-extracted parameters. + let (effective, query_hash, auto_params) = normalize_cypher(cypher_bytes); + let cached = store + .get_graph(graph_name) + .and_then(|g| g.plan_cache.lock().get(query_hash)); + + let (plan, auto_params) = match cached { + Some(cached) if !cached.read_only => (cached.plan, auto_params), + // Read-only hit on the write handler is a dispatcher anomaly — + // treat as a miss so classification runs as before. Same for a + // genuine miss. + _ => { + let (query, query_hash, auto_params) = + match parse_effective(cypher_bytes, &effective, query_hash, auto_params) { + Ok(t) => t, + Err(msg) => return Frame::Error(Bytes::from(msg)), + }; + let plan = match cypher::planner::compile(&query) { + Ok(p) => std::sync::Arc::new(p), + Err(e) => { + let msg = format!("ERR Cypher plan error: {e}"); + return Frame::Error(Bytes::from(msg)); + } + }; + if let Some(g) = store.get_graph(graph_name) { + g.plan_cache + .lock() + .insert(query_hash, plan.clone(), query.is_read_only()); + } + (plan, auto_params) } }; @@ -634,14 +889,6 @@ pub fn graph_query_write(store: &mut GraphStore, args: &[Frame]) -> Frame { Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), } - let plan = match cypher::planner::compile(&query) { - Ok(p) => p, - Err(e) => { - let msg = format!("ERR Cypher plan error: {e}"); - return Frame::Error(Bytes::from(msg)); - } - }; - let lsn = store.allocate_lsn(); // Scoped borrow: get mutable graph, execute, release borrow before WAL push. @@ -651,7 +898,12 @@ pub fn graph_query_write(store: &mut GraphStore, args: &[Frame]) -> Frame { None => return Frame::Error(Bytes::from_static(b"ERR graph not found")), }; - let params = parse_params(args); + // Merge auto-extracted literal values: the plan was compiled from + // the normalized text, so `$__pN` parameters must resolve. + let mut params = parse_params(args); + for (name, value) in auto_params { + params.insert(name, value); + } match cypher::executor::execute_mut(graph, &plan, ¶ms, lsn) { Ok(r) => r, Err(e) => { @@ -700,19 +952,58 @@ pub fn graph_query_write(store: &mut GraphStore, args: &[Frame]) -> Frame { properties.as_ref(), )); } - // Phase 174 FIX-01: SET/DELETE/MERGE records are only relevant - // for TXN rollback (handled in graph_query_or_write). The non-txn - // path here does not need WAL records for these — the write_buf - // mutation is already durable via the forward WAL. - cypher::executor::MutationRecord::SetProperty { .. } - | cypher::executor::MutationRecord::DeleteNode { .. } - | cypher::executor::MutationRecord::DeleteEdge { .. } => {} + // W2-2: DELETEs must be WAL-logged or the entity RESURRECTS at + // restart (replay re-adds it from its CreateNode/AddNode record; + // frozen-tier tombstones are pure write-buf state otherwise). + cypher::executor::MutationRecord::DeleteNode { node_id, .. } => { + store + .wal_pending + .push(crate::graph::wal::serialize_remove_node( + graph_name, *node_id, + )); + } + cypher::executor::MutationRecord::DeleteEdge { edge_id, .. } => { + store + .wal_pending + .push(crate::graph::wal::serialize_remove_edge( + graph_name, *edge_id, + )); + } + // W2-9: SET must be WAL-logged or a restart replays the original + // ADDNODE property state (the crash suite's G1/G3 caught exactly + // this loss). + cypher::executor::MutationRecord::SetProperty { + entity_id, + is_node, + key, + new_value, + .. + } => { + store + .wal_pending + .push(crate::graph::wal::serialize_set_prop( + graph_name, *entity_id, *is_node, *key, new_value, + )); + } + cypher::executor::MutationRecord::SetLabel { node_id, label } => { + store + .wal_pending + .push(crate::graph::wal::serialize_set_label( + graph_name, *node_id, *label, + )); + } } } // Bump version if any mutations were executed (Cypher write query). + // Task #32: also invalidate the graph's cached query results -- gated + // on `!result.mutations.is_empty()` so an idempotent MERGE match-branch + // (zero mutation records) doesn't pay an invalidation for a no-op. if !result.mutations.is_empty() { store.bump_version(); + if let Some(graph) = store.get_graph_mut(graph_name) { + graph.touch(); + } } exec_result_to_frame(&result) @@ -771,31 +1062,61 @@ pub fn graph_query_or_write( // Raw-hash pre-lookup (Fix 2): an EXACT repeat of a query text we've // already compiled hits here without ever calling `parameterize()`. + // W2-7: a read-only hit routes to the read path, a WRITE hit to the + // write path — both with zero parse/compile work. let raw_hash = cypher::planner::hash_query(cypher_bytes); - if let Some(graph) = store.get_graph(graph_name) { - if let Some(cached) = graph.plan_cache.lock().get(raw_hash) { - let auto_params = cached.auto_params.as_ref().clone(); + let raw_cached = store + .get_graph(graph_name) + .and_then(|g| g.plan_cache.lock().get(raw_hash)); + if let Some(cached) = raw_cached { + let auto_params = cached.auto_params.as_ref().clone(); + if cached.read_only { + let Some(graph) = store.get_graph(graph_name) else { + return ( + Frame::Error(Bytes::from_static(b"ERR graph not found")), + Vec::new(), + Vec::new(), + ); + }; return ( - run_read_query(graph, args, &cached.plan, &cached.slots, auto_params), + // Task #32: this auto-routing entry point is NOT wired into + // the result cache (`None`) -- see module docs for the + // scope boundary (protocol_version is not reliably + // available at every caller of `graph_query_or_write`, + // e.g. the cross-shard `ShardMessage::GraphCommand` hop). + run_read_query(graph, args, &cached.plan, &cached.slots, auto_params, None), Vec::new(), Vec::new(), ); } + return execute_write_plan(store, graph_name, args, &cached.plan, auto_params); } let (effective, query_hash, auto_params) = normalize_cypher(cypher_bytes); - // Fast path: normalized-hash plan-cache hit ⇒ read-only plan (PlanCache - // invariant) ⇒ route to the read path with ZERO parse/compile work. - if let Some(graph) = store.get_graph(graph_name) { - let cached = graph.plan_cache.lock().get(query_hash); - if let Some(cached) = cached { + // Fast path: normalized-hash plan-cache hit — a read-only plan routes to + // the read path, a WRITE plan (W2-7) to the write path; both with ZERO + // parse/compile work (per-run literal values arrive via the auto-params). + let cached = store + .get_graph(graph_name) + .and_then(|g| g.plan_cache.lock().get(query_hash)); + if let Some(cached) = cached { + if cached.read_only { + let Some(graph) = store.get_graph(graph_name) else { + return ( + Frame::Error(Bytes::from_static(b"ERR graph not found")), + Vec::new(), + Vec::new(), + ); + }; return ( - run_read_query(graph, args, &cached.plan, &cached.slots, auto_params), + // Task #32: see the raw-hash branch above -- not wired here. + run_read_query(graph, args, &cached.plan, &cached.slots, auto_params, None), Vec::new(), Vec::new(), ); } + return execute_write_plan(store, graph_name, args, &cached.plan, auto_params); } // Slow path: parse once (normalized text, raw fallback) and classify. @@ -832,188 +1153,256 @@ pub fn graph_query_or_write( query_hash, plan.clone(), auto_params.clone(), + true, ); ( - run_read_query(graph, args, &plan, &slots, auto_params), + // Task #32: see the raw-hash branch above -- not wired here. + run_read_query(graph, args, &plan, &slots, auto_params, None), Vec::new(), Vec::new(), ) } else { - // Decay biases read-path traversal cost only; a write query must not - // silently accept (or skip validating) the flag. Reject before any - // side effect (LSN allocation, mutation). - match parse_decay(args) { - Ok(None) => {} - Ok(Some(_)) => { - return ( - Frame::Error(Bytes::from_static( - b"ERR --decay requires a read-only Cypher query", - )), - Vec::new(), - Vec::new(), - ); + let plan = match cypher::planner::compile(&query) { + Ok(p) => std::sync::Arc::new(p), + Err(e) => { + let msg = format!("ERR Cypher plan error: {e}"); + return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()); } - Err(msg) => { + }; + // W2-7: cache the write plan (flagged) so the next occurrence of + // this normalized text — or an exact repeat via the raw-hash + // pre-lookup — skips parse + compile entirely. + if let Some(g) = store.get_graph(graph_name) { + let _ = g.plan_cache.lock().insert_both( + raw_hash, + query_hash, + plan.clone(), + auto_params.clone(), + false, + ); + } + execute_write_plan(store, graph_name, args, &plan, auto_params) + } +} + +/// Write-execution tail shared by `graph_query_or_write`'s compile path and +/// its W2-7 plan-cache hit path: decay validation (before any side effect), +/// LSN allocation, `execute_mut` with auto-params merged, and the +/// mutation → WAL / txn-intent / undo-op fan-out. +fn execute_write_plan( + store: &mut GraphStore, + graph_name: &[u8], + args: &[Frame], + plan: &cypher::PhysicalPlan, + auto_params: Vec<(String, cypher::executor::Value)>, +) -> ( + Frame, + Vec, + Vec, +) { + // Decay biases read-path traversal cost only; a write query must not + // silently accept (or skip validating) the flag. Reject before any + // side effect (LSN allocation, mutation). + match parse_decay(args) { + Ok(None) => {} + Ok(Some(_)) => { + return ( + Frame::Error(Bytes::from_static( + b"ERR --decay requires a read-only Cypher query", + )), + Vec::new(), + Vec::new(), + ); + } + Err(msg) => { + return ( + Frame::Error(Bytes::from_static(msg.as_bytes())), + Vec::new(), + Vec::new(), + ); + } + } + + let lsn = store.allocate_lsn(); + + // Phase 174 FIX-02: extract mutations regardless of Ok/Err so that + // partial writes from before the error are visible to TXN.ABORT. + let (result_or_err, mutations) = { + let graph = match store.get_graph_mut(graph_name) { + Some(g) => g, + None => { return ( - Frame::Error(Bytes::from_static(msg.as_bytes())), + Frame::Error(Bytes::from_static(b"ERR graph not found")), Vec::new(), Vec::new(), ); } - } + }; - // Write path: compile plan (no cache for writes), execute with mutations. - let plan = match cypher::planner::compile(&query) { - Ok(p) => p, + // Merge auto-extracted literal values: the write plan was + // compiled from the normalized text, so `$__pN` parameters must + // resolve or CREATE would store Nulls. + let mut params = parse_params(args); + for (name, value) in auto_params { + params.insert(name, value); + } + match cypher::executor::execute_mut(graph, plan, ¶ms, lsn) { + Ok(r) => { + let muts = r.mutations; + ( + Ok(cypher::executor::ExecResult { + columns: r.columns, + rows: r.rows, + nodes_created: r.nodes_created, + nodes_deleted: r.nodes_deleted, + properties_set: r.properties_set, + execution_time_us: r.execution_time_us, + mutations: Vec::new(), // moved out above + }), + muts, + ) + } Err(e) => { - let msg = format!("ERR Cypher plan error: {e}"); - return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()); + let msg = format!("ERR Cypher execution error: {e}"); + (Err(msg), e.partial_mutations) } - }; + } + }; - let lsn = store.allocate_lsn(); - - // Phase 174 FIX-02: extract mutations regardless of Ok/Err so that - // partial writes from before the error are visible to TXN.ABORT. - let (result_or_err, mutations) = { - let graph = match store.get_graph_mut(graph_name) { - Some(g) => g, - None => { - return ( - Frame::Error(Bytes::from_static(b"ERR graph not found")), - Vec::new(), - Vec::new(), - ); - } - }; + // Phase 167: collect write intents for CrossStoreTxn rollback. Every + // CreateNode/CreateEdge mutation (from CreatePattern and the Merge + // create-branch) becomes an intent; MERGE match-branches produce no + // mutation and therefore no intent (idempotent rollback). + let mut intents: Vec = Vec::with_capacity(mutations.len()); + + // Phase 174 FIX-01: collect undo ops for SET/DELETE/MERGE rollback. + let gname_bytes = Bytes::copy_from_slice(graph_name); + let mut undo_ops: Vec = Vec::new(); - // Merge auto-extracted literal values: the write plan was - // compiled from the normalized text, so `$__pN` parameters must - // resolve or CREATE would store Nulls. - let mut params = parse_params(args); - for (name, value) in auto_params { - params.insert(name, value); + // Generate WAL records for mutations + collect intents/undo ops. + for mutation in &mutations { + match mutation { + cypher::executor::MutationRecord::CreateNode { + node_id, + labels, + properties, + embedding, + } => { + intents.push(cypher::executor::GraphWriteIntent { + entity_id: *node_id, + is_node: true, + }); + store + .wal_pending + .push(crate::graph::wal::serialize_add_node( + graph_name, + *node_id, + labels, + properties, + embedding.as_deref(), + )); } - match cypher::executor::execute_mut(graph, &plan, ¶ms, lsn) { - Ok(r) => { - let muts = r.mutations; - ( - Ok(cypher::executor::ExecResult { - columns: r.columns, - rows: r.rows, - nodes_created: r.nodes_created, - nodes_deleted: r.nodes_deleted, - properties_set: r.properties_set, - execution_time_us: r.execution_time_us, - mutations: Vec::new(), // moved out above - }), - muts, - ) - } - Err(e) => { - let msg = format!("ERR Cypher execution error: {e}"); - (Err(msg), e.partial_mutations) - } + cypher::executor::MutationRecord::CreateEdge { + edge_id, + src_id, + dst_id, + edge_type, + weight, + properties, + } => { + intents.push(cypher::executor::GraphWriteIntent { + entity_id: *edge_id, + is_node: false, + }); + store + .wal_pending + .push(crate::graph::wal::serialize_add_edge( + graph_name, + *edge_id, + *src_id, + *dst_id, + *edge_type, + *weight, + properties.as_ref(), + )); } - }; - - // Phase 167: collect write intents for CrossStoreTxn rollback. Every - // CreateNode/CreateEdge mutation (from CreatePattern and the Merge - // create-branch) becomes an intent; MERGE match-branches produce no - // mutation and therefore no intent (idempotent rollback). - let mut intents: Vec = - Vec::with_capacity(mutations.len()); - - // Phase 174 FIX-01: collect undo ops for SET/DELETE/MERGE rollback. - let gname_bytes = Bytes::copy_from_slice(graph_name); - let mut undo_ops: Vec = Vec::new(); - - // Generate WAL records for mutations + collect intents/undo ops. - for mutation in &mutations { - match mutation { - cypher::executor::MutationRecord::CreateNode { - node_id, - labels, - properties, - embedding, - } => { - intents.push(cypher::executor::GraphWriteIntent { - entity_id: *node_id, - is_node: true, - }); - store - .wal_pending - .push(crate::graph::wal::serialize_add_node( - graph_name, - *node_id, - labels, - properties, - embedding.as_deref(), - )); - } - cypher::executor::MutationRecord::CreateEdge { - edge_id, - src_id, - dst_id, - edge_type, - weight, - properties, - } => { - intents.push(cypher::executor::GraphWriteIntent { - entity_id: *edge_id, - is_node: false, - }); - store - .wal_pending - .push(crate::graph::wal::serialize_add_edge( - graph_name, - *edge_id, - *src_id, - *dst_id, - *edge_type, - *weight, - properties.as_ref(), - )); - } - // Phase 174 FIX-01: new variants for SET/DELETE/MERGE rollback. - cypher::executor::MutationRecord::SetProperty { - entity_id, - is_node, - key, - old_value, - } => { - undo_ops.push(crate::transaction::GraphUndoOp::RestoreProperty { - graph_name: gname_bytes.clone(), - entity_id: *entity_id, - is_node: *is_node, - prop_key: *key, - old_value: old_value.clone(), - }); - } - cypher::executor::MutationRecord::DeleteNode { node_id, .. } => { - undo_ops.push(crate::transaction::GraphUndoOp::UndeleteNode { - graph_name: gname_bytes.clone(), - node_id: *node_id, - delete_lsn: lsn, - }); - } - cypher::executor::MutationRecord::DeleteEdge { edge_id, .. } => { - undo_ops.push(crate::transaction::GraphUndoOp::UndeleteEdge { - graph_name: gname_bytes.clone(), - edge_id: *edge_id, - }); - } + // Phase 174 FIX-01: new variants for SET/DELETE/MERGE rollback. + cypher::executor::MutationRecord::SetProperty { + entity_id, + is_node, + key, + old_value, + new_value, + } => { + undo_ops.push(crate::transaction::GraphUndoOp::RestoreProperty { + graph_name: gname_bytes.clone(), + entity_id: *entity_id, + is_node: *is_node, + prop_key: *key, + old_value: old_value.clone(), + }); + // W2-9: WAL the SET or it is silently lost on kill -9 (replay + // re-runs the original ADDNODE property state). + store + .wal_pending + .push(crate::graph::wal::serialize_set_prop( + graph_name, *entity_id, *is_node, *key, new_value, + )); + } + // W2-9: WAL-only — label rollback was never captured (pre-existing + // Phase 174 scope), so no undo op here. + cypher::executor::MutationRecord::SetLabel { node_id, label } => { + store + .wal_pending + .push(crate::graph::wal::serialize_set_label( + graph_name, *node_id, *label, + )); + } + cypher::executor::MutationRecord::DeleteNode { node_id, .. } => { + undo_ops.push(crate::transaction::GraphUndoOp::UndeleteNode { + graph_name: gname_bytes.clone(), + node_id: *node_id, + delete_lsn: lsn, + }); + // W2-2: WAL the delete or it resurrects at restart. + store + .wal_pending + .push(crate::graph::wal::serialize_remove_node( + graph_name, *node_id, + )); + } + cypher::executor::MutationRecord::DeleteEdge { edge_id, .. } => { + undo_ops.push(crate::transaction::GraphUndoOp::UndeleteEdge { + graph_name: gname_bytes.clone(), + edge_id: *edge_id, + }); + store + .wal_pending + .push(crate::graph::wal::serialize_remove_edge( + graph_name, *edge_id, + )); } } + } - // Phase 174 FIX-02: return intents/undo_ops on BOTH Ok and Err paths - // so TXN.ABORT can roll back partial writes from before the error. - match result_or_err { - Ok(ref result) => (exec_result_to_frame(result), intents, undo_ops), - Err(msg) => (Frame::Error(Bytes::from(msg)), intents, undo_ops), + // Task #32: invalidate the graph's cached query results whenever this + // write actually produced mutations (mirrors the `graph_query_write` + // gate -- an idempotent MERGE match-branch must not pay an + // invalidation for a no-op). Runs on both Ok and Err (Phase 174 FIX-02: + // `mutations` already includes partial pre-error writes). + if !mutations.is_empty() { + if let Some(graph) = store.get_graph_mut(graph_name) { + graph.touch(); } } + + // Phase 174 FIX-02: return intents/undo_ops on BOTH Ok and Err paths + // so TXN.ABORT can roll back partial writes from before the error. + match result_or_err { + Ok(ref result) => (exec_result_to_frame(result), intents, undo_ops), + Err(msg) => (Frame::Error(Bytes::from(msg)), intents, undo_ops), + } } /// GRAPH.RO_QUERY @@ -1021,13 +1410,13 @@ pub fn graph_query_or_write( /// Like GRAPH.QUERY but rejects write clauses (CREATE, DELETE, SET, MERGE). /// Shares the single-parse read core with GRAPH.QUERY — a plan-cache hit is /// read-only by the cache invariant, so no classification re-parse is needed. -pub fn graph_ro_query(store: &GraphStore, args: &[Frame]) -> Frame { +pub fn graph_ro_query(store: &GraphStore, args: &[Frame], protocol_version: Option) -> Frame { if args.len() < 2 { return Frame::Error(Bytes::from_static( b"ERR wrong number of arguments for 'GRAPH.RO_QUERY' command", )); } - graph_query_readonly(store, args, true) + graph_query_readonly(store, args, true, protocol_version) } /// GRAPH.EXPLAIN @@ -1149,10 +1538,14 @@ pub fn graph_profile(store: &GraphStore, args: &[Frame]) -> Frame { Ok(d) => d, Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), }; + let guard = match query_guard(args, 0) { + Ok(g) => g, + Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), + }; let ctx = cypher::executor::ExecutionContext { valid_time_as_of: valid_at, decay, - guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), + guard: Some(guard), ..Default::default() }; let profile = match cypher::executor::execute_profile(graph, &plan, ¶ms, &ctx) { @@ -1251,7 +1644,9 @@ fn value_to_frame(value: &cypher::executor::Value) -> Frame { Value::Null => Frame::Null, Value::Int(n) => Frame::Integer(*n), Value::Float(f) => Frame::Double(*f), - Value::String(s) => Frame::BulkString(Bytes::from(s.clone())), + // Zero-copy (W2-4): the stored property's Bytes flows straight into + // the reply frame — a refcount bump, not an allocation. + Value::String(s) => Frame::BulkString(s.clone()), Value::Bool(b) => Frame::Boolean(*b), Value::Node(key) => { // "node:" (5) + max u64 (20 digits) = 25 bytes max diff --git a/src/command/graph/graph_write.rs b/src/command/graph/graph_write.rs index d5f0623cf..fb0627c6e 100644 --- a/src/command/graph/graph_write.rs +++ b/src/command/graph/graph_write.rs @@ -183,6 +183,10 @@ pub fn graph_addnode(store: &mut GraphStore, args: &[Frame]) -> Frame { // Bump version AFTER successful node insert. store.bump_version(); + // Task #32: invalidate this graph's cached query results (write_gen). + if let Some(graph) = store.get_graph_mut(graph_name) { + graph.touch(); + } Frame::Integer(external_id as i64) } @@ -352,6 +356,12 @@ pub fn graph_addedge(store: &mut GraphStore, args: &[Frame]) -> Frame { // Check if compaction threshold reached after edge insertion. // If so, freeze the mutable MemGraph and convert to an immutable // CSR segment. This is synchronous and fast (<5ms at 64K edges). + // + // Task #32: freeze_and_compact is a storage-tier reorg with + // identical logical content -- it must NOT bump write_gen (that + // would flush the result cache every edge_threshold edges for + // zero correctness benefit). touch() below covers only the edge + // insert itself. let needs_compact = store .get_graph(graph_name) .is_some_and(|g| g.should_compact()); @@ -364,6 +374,9 @@ pub fn graph_addedge(store: &mut GraphStore, args: &[Frame]) -> Frame { // Bump version AFTER successful edge insert. store.bump_version(); + if let Some(graph) = store.get_graph_mut(graph_name) { + graph.touch(); + } Frame::Integer(external_id as i64) } Err(crate::graph::memgraph::GraphError::NodeNotFound) => Frame::Error(Bytes::from_static( diff --git a/src/command/graph/mod.rs b/src/command/graph/mod.rs index 5710c35e9..d9414986d 100644 --- a/src/command/graph/mod.rs +++ b/src/command/graph/mod.rs @@ -62,7 +62,18 @@ pub fn is_cypher_write_query(args: &[crate::protocol::Frame]) -> bool { } /// Dispatch read-only GRAPH.* commands. Takes &GraphStore (shared). -pub fn dispatch_graph_read(store: &GraphStore, cmd: &[u8], args: &[Frame]) -> Frame { +/// +/// `protocol_version` (Task #32): forwarded to GRAPH.QUERY / GRAPH.RO_QUERY +/// for the Cypher result cache. `None` when the caller cannot reliably +/// determine the originating connection's negotiated RESP version (see +/// `graph_query`'s doc comment) -- the result cache is simply not consulted +/// or populated for that call, never a correctness risk. +pub fn dispatch_graph_read( + store: &GraphStore, + cmd: &[u8], + args: &[Frame], + protocol_version: Option, +) -> Frame { if cmd.eq_ignore_ascii_case(b"GRAPH.NEIGHBORS") { graph_neighbors(store, args) } else if cmd.eq_ignore_ascii_case(b"GRAPH.INFO") { @@ -70,9 +81,9 @@ pub fn dispatch_graph_read(store: &GraphStore, cmd: &[u8], args: &[Frame]) -> Fr } else if cmd.eq_ignore_ascii_case(b"GRAPH.LIST") { graph_list(store) } else if cmd.eq_ignore_ascii_case(b"GRAPH.QUERY") { - graph_query(store, args) + graph_query(store, args, protocol_version) } else if cmd.eq_ignore_ascii_case(b"GRAPH.RO_QUERY") { - graph_ro_query(store, args) + graph_ro_query(store, args, protocol_version) } else if cmd.eq_ignore_ascii_case(b"GRAPH.EXPLAIN") { graph_explain(store, args) } else if cmd.eq_ignore_ascii_case(b"GRAPH.PROFILE") { @@ -152,7 +163,11 @@ pub fn dispatch_graph_command(store: &mut GraphStore, command: &Frame) -> Frame if is_graph_write_cmd(cmd) { dispatch_graph_write(store, cmd, args) } else { - dispatch_graph_read(store, cmd, args) + // Task #32: `None` -- this dispatch path (cross-shard GraphCommand / + // handler_single) has no reliable access to the originating + // connection's negotiated protocol_version, so the result cache is + // not consulted here. See `graph_query`'s doc comment. + dispatch_graph_read(store, cmd, args, None) } } @@ -172,7 +187,11 @@ pub fn dispatch_graph_cmd_args(store: &mut GraphStore, cmd: &[u8], args: &[Frame if is_graph_write_cmd(cmd) { dispatch_graph_write(store, cmd, args) } else { - dispatch_graph_read(store, cmd, args) + // Task #32: `None` -- this dispatch path (cross-shard GraphCommand / + // handler_single) has no reliable access to the originating + // connection's negotiated protocol_version, so the result cache is + // not consulted here. See `graph_query`'s doc comment. + dispatch_graph_read(store, cmd, args, None) } } @@ -224,6 +243,40 @@ mod tests { .collect() } + #[test] + fn test_parse_timeout_ms() { + let args = |parts: &[&[u8]]| -> Vec { + parts + .iter() + .map(|p| Frame::BulkString(Bytes::from(p.to_vec()))) + .collect() + }; + // Absent keyword → None. + assert_eq!( + graph_read::parse_timeout_ms(&args(&[b"g", b"MATCH (n) RETURN n"])), + Ok(None) + ); + // Present with value (case-insensitive keyword). + assert_eq!( + graph_read::parse_timeout_ms(&args(&[b"g", b"q", b"timeout", b"250"])), + Ok(Some(250)) + ); + // 0 is valid (= unlimited). + assert_eq!( + graph_read::parse_timeout_ms(&args(&[b"g", b"q", b"TIMEOUT", b"0"])), + Ok(Some(0)) + ); + // Garbage values are errors, not silent no-ops. + for bad in [&b"abc"[..], b"-5", b"1.5", b""] { + assert!( + graph_read::parse_timeout_ms(&args(&[b"g", b"q", b"TIMEOUT", bad])).is_err(), + "TIMEOUT {bad:?} must be rejected" + ); + } + // Dangling keyword is an error. + assert!(graph_read::parse_timeout_ms(&args(&[b"g", b"q", b"TIMEOUT"])).is_err()); + } + #[test] fn test_plan_cache_shared_across_literal_variants() { let mut store = GraphStore::new(); @@ -270,11 +323,15 @@ mod tests { ); let graph = store.get_graph(b"g").expect("graph exists"); + // Dual-key cache: each shape stores the first variant's raw-text hash + // plus the shared literal-normalized hash (2 entries/shape); the + // second variant hits the normalized entry, adding nothing. One WRITE + // plan for both CREATEs (W2-7) + one READ plan for both MATCHes. assert_eq!( - graph.plan_cache.lock().distinct_plan_count(), - 1, - "queries differing only in literal values must share one cached plan \ - (and write queries must not be cached)" + graph.plan_cache.lock().len(), + 4, + "literal variants must share one cached plan per shape \ + (raw + normalized key each for the CREATE and MATCH shapes)" ); } @@ -324,10 +381,14 @@ mod tests { ); let graph = store.get_graph(b"g").expect("graph exists"); + // Dual-key cache: 2 entries per shape (first variant's raw hash + + // shared normalized hash) — see test_plan_cache_shared_across_ + // literal_variants for the breakdown. assert_eq!( - graph.plan_cache.lock().distinct_plan_count(), - 1, - "string-literal variants must share one cached plan" + graph.plan_cache.lock().len(), + 4, + "string-literal variants must share one cached plan per shape \ + (one write, one read — W2-7 caches writes too)" ); } diff --git a/src/command/temporal.rs b/src/command/temporal.rs index 3747297c5..22ed45cfe 100644 --- a/src/command/temporal.rs +++ b/src/command/temporal.rs @@ -125,6 +125,13 @@ pub fn apply_invalidate( entity_id, is_node, wall_ms, wall_ms, ); gs.wal_pending.push(payload); + // Task #32: TEMPORAL.INVALIDATE mutates valid_to on write_buf directly, + // changing what a subsequent read sees -- invalidate the graph's cached + // query results. Re-fetch rather than reuse `named_graph` above: the + // borrow was released at the `let Some(named_graph) = ...` match end. + if let Some(named_graph) = gs.get_graph_mut(graph_name) { + named_graph.touch(); + } Ok(()) } diff --git a/src/command/vector_search/tests.rs b/src/command/vector_search/tests.rs index 876525576..753681c46 100644 --- a/src/command/vector_search/tests.rs +++ b/src/command/vector_search/tests.rs @@ -1464,6 +1464,76 @@ fn test_ft_config_unknown_index() { ); } +#[test] +fn test_insert_path_triggers_background_compact_without_search() { + let _metrics_guard = METRICS_LOCK.read(); + let mut store = VectorStore::new(); + let mut text = crate::text::store::TextStore::new(); + + // COMPACT_THRESHOLD 100 (the minimum) so a modest bulk load crosses it. + let args = vec![ + bulk(b"autoidx"), + bulk(b"ON"), + bulk(b"HASH"), + bulk(b"PREFIX"), + bulk(b"1"), + bulk(b"doc:"), + bulk(b"SCHEMA"), + bulk(b"vec"), + bulk(b"VECTOR"), + bulk(b"HNSW"), + bulk(b"8"), + bulk(b"TYPE"), + bulk(b"FLOAT32"), + bulk(b"DIM"), + bulk(b"8"), + bulk(b"DISTANCE_METRIC"), + bulk(b"L2"), + bulk(b"COMPACT_THRESHOLD"), + bulk(b"100"), + ]; + let result = ft_create(&mut store, &mut text, &args); + assert!(matches!(result, Frame::SimpleString(_)), "{result:?}"); + + let hset = |store: &mut VectorStore, text: &mut _, i: usize| { + let key = format!("doc:{i}"); + let vec_bytes: Vec = (0..8u32) + .flat_map(|d| ((i as f32) * 0.37 + d as f32).to_le_bytes()) + .collect(); + let hset_args = vec![bulk(key.as_bytes()), bulk(b"vec"), bulk(&vec_bytes)]; + crate::shard::spsc_handler::auto_index_hset_public(store, text, key.as_bytes(), &hset_args); + }; + + // Pure bulk load: ONLY the HSET auto-index hook runs. No FT.SEARCH, no + // FT.COMPACT, no autovacuum tick — before the insert-path trigger this + // left every vector in the brute-force mutable tier indefinitely. + for i in 0..150 { + hset(&mut store, &mut text, i); + } + + // The background worker builds asynchronously; each further insert polls + // installs (same as the search path). Nudge until the immutable segment + // lands or we time out. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + let mut extra = 150usize; + let mut installed = false; + while std::time::Instant::now() < deadline { + hset(&mut store, &mut text, extra); + extra += 1; + #[allow(clippy::unwrap_used)] // index created above; unit-test context + let idx = store.get_index(b"autoidx").unwrap(); + if !idx.segments.load().immutable.is_empty() { + installed = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!( + installed, + "bulk insert never dispatched+installed a background compaction (insert-path trigger missing)" + ); +} + #[test] fn test_ft_config_autocompact_guards_try_compact() { let _metrics_guard = METRICS_LOCK.read(); diff --git a/src/config.rs b/src/config.rs index f27356cc3..b341a5198 100644 --- a/src/config.rs +++ b/src/config.rs @@ -619,6 +619,18 @@ pub struct ServerConfig { #[arg(long = "graph-dead-edge-trigger", default_value_t = 0.20)] pub graph_dead_edge_trigger: f64, + /// Default graph traversal timeout in milliseconds (0 = unlimited). + /// + /// Bounds how long a single Cypher traversal (variable-length expand, + /// shortestPath, GRAPH.TRAVERSE hop loop) may run — and therefore how long + /// it may hold a graph snapshot. A per-query `TIMEOUT ` argument on + /// GRAPH.QUERY / GRAPH.RO_QUERY / GRAPH.PROFILE / GRAPH.TRAVERSE overrides + /// this for one query (RedisGraph parity). + /// + /// Default: 30_000 (30 s). + #[arg(long = "graph-timeout-ms", default_value_t = 30_000)] + pub graph_timeout_ms: u64, + // ── MA4: Weighted compaction scheduling ─────────────────────────────── /// Minimum seconds before a stale entity is forced to be scheduled by the /// autovacuum daemon regardless of its compaction weight (anti-starvation cap). diff --git a/src/graph/compaction.rs b/src/graph/compaction.rs index 1e261d351..aba7df324 100644 --- a/src/graph/compaction.rs +++ b/src/graph/compaction.rs @@ -572,6 +572,12 @@ pub fn compact_segments( incoming: std::sync::OnceLock::new(), props_index: std::sync::OnceLock::new(), hnsw_bridge: std::sync::OnceLock::new(), + hnsw_building: std::sync::atomic::AtomicBool::new(false), + // GraphUnion merge does NOT remap/merge posting row ids across + // input segments (pre-approved scope decision, see + // `text_index.rs` module docs) -- the merged segment starts empty + // and rebuilds its text index lazily, from scratch, on first use. + text_index: std::sync::OnceLock::new(), }) } @@ -725,6 +731,100 @@ mod tests { CsrSegment::from_frozen(frozen, lsn).expect("ok") } + /// Like [`make_csr`] but each node carries a string `bio` property + /// (P3 design part B: `GraphUnion` merge text-index correctness test). + /// `id_base` keeps external node ids disjoint across segments being + /// merged in the same test -- two independently-created `MemGraph`s + /// both start id allocation at 0, which would silently ALIAS nodes + /// across segments during `compact_segments`'s `external_id`-keyed + /// dedup (see `view.rs`'s module docs on this exact hazard for the + /// production id-allocation path). + fn make_csr_with_bios( + id_base: u64, + bios: &[&str], + edges: &[(usize, usize)], + lsn: u64, + ) -> CsrSegment { + let mut mg = MemGraph::new(100_000); + let bio_pid = crate::command::graph::graph_write::label_to_id(b"bio"); + let mut keys = Vec::with_capacity(bios.len()); + for (i, bio) in bios.iter().enumerate() { + let props = smallvec![( + bio_pid, + crate::graph::types::PropertyValue::String(bytes::Bytes::copy_from_slice( + bio.as_bytes() + )), + )]; + keys.push(mg.add_node_with_id(id_base + i as u64, smallvec![0], props, None, 1)); + } + for &(s, d) in edges { + mg.add_edge(keys[s], keys[d], 1, 1.0, None, 2).expect("ok"); + } + let frozen = mg.freeze().expect("ok"); + CsrSegment::from_frozen(frozen, lsn).expect("ok") + } + + /// P3 design part B, B1 test #11 (the acceptance gate for the + /// GraphUnion scope decision documented in `text_index.rs`): a merged + /// segment must NOT carry over a stale/unmapped text index (there is + /// none to carry -- posting row ids are never remapped) and its + /// FRESHLY lazily-built `SegmentTextIndex` must be correct over the + /// merged row space, covering every node from every input segment. + #[test] + fn test_graph_union_merge_rebuilds_text_index_lazily_and_correctly() { + // seg1: node 0 matches `CONTAINS 'rust'` by an exact token; node 1 + // has no bio text relevant to the query. + let seg1 = Arc::new(CsrStorage::from(make_csr_with_bios( + 0, + &["loves rust", "no match here"], + &[(0, 1)], + 10, + ))); + // seg2: node 0's bio is the substring-across-token-boundary crux + // ("trusted" contains "rust" as a raw substring but is a DIFFERENT + // token); node 1 has no relevant text either. + let seg2 = Arc::new(CsrStorage::from(make_csr_with_bios( + 100, + &["trusted friend", "unrelated text"], + &[(0, 1)], + 20, + ))); + + let config = CompactionConfig { + min_segments: 2, + max_segment_edges: 1_000_000, + }; + let merged = compact_segments(&[seg1, seg2], &config).expect("merge ok"); + assert_eq!( + merged.node_count(), + 4, + "all 4 distinct external ids survive the merge" + ); + + // Scope decision: no posting-row remap -- the merged segment starts + // with an empty text-index cell. + assert!( + merged.text_index.get().is_none(), + "GraphUnion merge must not carry over a stale/pre-merge text index" + ); + + let merged_storage = CsrStorage::from(merged); + let bio_pid = crate::command::graph::graph_write::label_to_id(b"bio"); + let candidates = merged_storage + .text_index() + .candidate_rows(bio_pid) + .cloned() + .unwrap_or_default(); + // Every merged node carries a string bio -- the presence bitmap + // (the correctness-critical SUPERSET source) must cover all 4, + // including the substring-crux row from seg2. + assert_eq!( + candidates.len(), + 4, + "candidates must cover every merged row, got {candidates:?}" + ); + } + #[test] fn test_merge_three_segments() { let seg1 = Arc::new(CsrStorage::from(make_csr(3, &[(0, 1), (1, 2)], 10))); diff --git a/src/graph/cross_shard.rs b/src/graph/cross_shard.rs deleted file mode 100644 index 852171679..000000000 --- a/src/graph/cross_shard.rs +++ /dev/null @@ -1,480 +0,0 @@ -//! Cross-shard graph traversal via scatter-gather over SPSC mesh. -//! -//! When a graph traversal encounters nodes that may live on other shards, -//! the coordinator groups node IDs by target shard, sends `GraphTraverse` -//! messages via SPSC, and merges results from all participating shards. -//! -//! **Shard-local optimization:** If a graph name contains a hash tag -//! `{partition_key}`, all GRAPH.* operations route to the same shard and -//! no cross-shard traversal is needed. -//! -//! **Depth limit:** Cross-shard expansion stops at a configurable depth -//! (default 2 hops) and returns partial results with a truncation notice. -//! -//! **Snapshot consistency:** The originating traversal's snapshot-LSN is -//! forwarded to all participating shards so they all see the same graph version. - -use bytes::Bytes; -use slotmap::Key; - -use crate::graph::store::GraphStore; -use crate::graph::types::Direction; -use crate::protocol::Frame; - -/// Default maximum cross-shard hops before truncation. -pub const DEFAULT_CROSS_SHARD_DEPTH_LIMIT: u32 = 2; - -/// Result of a local shard expansion for cross-shard traversal. -/// -/// Contains neighbor node external IDs discovered on this shard, -/// plus the edge/node Frame representations for the final response. -#[derive(Debug)] -pub struct TraversalShardResult { - /// External IDs of discovered neighbor nodes. - pub neighbor_ids: Vec, - /// Edge type for each discovered neighbor (parallel with neighbor_ids). - pub edge_types: Vec, - /// RESP3 frames for edges and nodes found on this shard. - pub frames: Vec, - /// Whether the result was truncated due to depth or size limits. - pub truncated: bool, -} - -/// Handle an incoming `GraphTraverse` SPSC message on this shard. -/// -/// Expands the given node IDs locally using the shard's MemGraph, -/// returning a Frame::Array of neighbor edges and nodes, plus a -/// bulk string listing discovered neighbor external IDs for further -/// expansion by the coordinator. -/// -/// Response format: -/// ```text -/// Array [ -/// BulkString("NEIGHBORS"), -- marker -/// Array [ ... frames ... ], -- edge/node RESP3 maps -/// BulkString("DISCOVERED"), -- marker -/// Array [ Integer(id), ... ], -- neighbor external IDs for next hop -/// BulkString("TRUNCATED"), -- only present if truncated -/// ] -/// ``` -pub fn handle_graph_traverse( - store: &GraphStore, - graph_name: &[u8], - node_ids: &[u64], - edge_type_filter: Option, - snapshot_lsn: u64, -) -> Frame { - let graph = match store.get_graph(graph_name) { - Some(g) => g, - None => { - return Frame::Error(Bytes::from_static( - b"ERR graph not found for cross-shard traversal", - )); - } - }; - - let memgraph = &graph.write_buf; - let max_results = 10_000usize; - let mut result_frames: Vec = Vec::with_capacity(64); - let mut discovered_ids: Vec = Vec::with_capacity(64); - let mut visited_this_batch = std::collections::HashSet::new(); - let mut truncated = false; - - for &ext_id in node_ids { - let node_key = crate::command::graph::graph_write::external_id_to_node_key(ext_id); - - // Skip nodes that don't exist on this shard. - if memgraph.get_node(node_key).is_none() { - continue; - } - - for (edge_key, neighbor_key) in memgraph.neighbors(node_key, Direction::Both, snapshot_lsn) - { - // Apply edge type filter. - if let Some(filter) = edge_type_filter { - if let Some(edge) = memgraph.get_edge(edge_key) { - if edge.edge_type != filter { - continue; - } - } - } - - let neighbor_ext_id = neighbor_key.data().as_ffi(); - - if !visited_this_batch.insert(neighbor_ext_id) { - continue; - } - - // Add edge frame. - if let Some(edge) = memgraph.get_edge(edge_key) { - result_frames.push(edge_to_traverse_frame(edge_key, edge)); - } - - // Add node frame. - if let Some(node) = memgraph.get_node(neighbor_key) { - result_frames.push(node_to_traverse_frame(neighbor_key, node)); - } - - // Track discovered IDs for further expansion. - discovered_ids.push(Frame::Integer(neighbor_ext_id as i64)); - - if result_frames.len() >= max_results { - truncated = true; - break; - } - } - - if truncated { - break; - } - } - - // Build response array. - let mut response = Vec::with_capacity(5); - response.push(Frame::BulkString(Bytes::from_static(b"NEIGHBORS"))); - response.push(Frame::Array(result_frames.into())); - response.push(Frame::BulkString(Bytes::from_static(b"DISCOVERED"))); - response.push(Frame::Array(discovered_ids.into())); - if truncated { - response.push(Frame::BulkString(Bytes::from_static(b"TRUNCATED"))); - } - - Frame::Array(response.into()) -} - -/// Check if a graph name has a hash tag, meaning all operations are shard-local. -/// -/// When a graph name like `{social}.friends` is used, the `{social}` tag -/// ensures all operations route to the same shard, so no cross-shard -/// traversal is ever needed. -#[inline] -pub fn graph_has_hash_tag(graph_name: &[u8]) -> bool { - crate::shard::dispatch::extract_hash_tag(graph_name).is_some() -} - -/// Parse a GraphTraverse response frame into structured data. -/// -/// Used by the scatter-gather coordinator to extract discovered node IDs -/// from the response for the next hop. -pub fn parse_traverse_response(frame: &Frame) -> Option { - let items = match frame { - Frame::Array(items) => items, - _ => return None, - }; - - if items.len() < 4 { - return None; - } - - // items[0] = "NEIGHBORS" marker - // items[1] = Array of edge/node frames - // items[2] = "DISCOVERED" marker - // items[3] = Array of discovered IDs - // items[4] = "TRUNCATED" (optional) - - let frames = match &items[1] { - Frame::Array(f) => f.to_vec(), - _ => return None, - }; - - let discovered = match &items[3] { - Frame::Array(ids) => ids, - _ => return None, - }; - - let neighbor_ids: Vec = discovered - .iter() - .filter_map(|f| match f { - Frame::Integer(id) => Some(*id as u64), - _ => None, - }) - .collect(); - - let truncated = - items.len() > 4 && matches!(&items[4], Frame::BulkString(b) if b.as_ref() == b"TRUNCATED"); - - Some(TraversalShardResult { - neighbor_ids, - edge_types: Vec::new(), // Edge types tracked in frames, not separately - frames, - truncated, - }) -} - -// --------------------------------------------------------------------------- -// Frame helpers (lightweight versions for traversal responses) -// --------------------------------------------------------------------------- - -fn node_to_traverse_frame( - key: crate::graph::types::NodeKey, - node: &crate::graph::types::MutableNode, -) -> Frame { - let external_id = key.data().as_ffi(); - - let labels: Vec = node - .labels - .iter() - .map(|&l| Frame::Integer(l as i64)) - .collect(); - - Frame::Map(vec![ - ( - Frame::SimpleString(Bytes::from_static(b"id")), - Frame::Integer(external_id as i64), - ), - ( - Frame::SimpleString(Bytes::from_static(b"labels")), - Frame::Array(labels.into()), - ), - ]) -} - -fn edge_to_traverse_frame( - key: crate::graph::types::EdgeKey, - edge: &crate::graph::types::MutableEdge, -) -> Frame { - let external_id = key.data().as_ffi(); - let src_ext = edge.src.data().as_ffi(); - let dst_ext = edge.dst.data().as_ffi(); - - Frame::Map(vec![ - ( - Frame::SimpleString(Bytes::from_static(b"id")), - Frame::Integer(external_id as i64), - ), - ( - Frame::SimpleString(Bytes::from_static(b"type")), - Frame::Integer(edge.edge_type as i64), - ), - ( - Frame::SimpleString(Bytes::from_static(b"src")), - Frame::Integer(src_ext as i64), - ), - ( - Frame::SimpleString(Bytes::from_static(b"dst")), - Frame::Integer(dst_ext as i64), - ), - ]) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::graph::store::GraphStore; - use crate::protocol::FrameVec; - - fn make_cmd(parts: &[&[u8]]) -> Frame { - let frames: Vec = parts - .iter() - .map(|p| Frame::BulkString(Bytes::from(p.to_vec()))) - .collect(); - Frame::Array(FrameVec::from_vec(frames)) - } - - #[test] - fn test_graph_has_hash_tag() { - assert!(graph_has_hash_tag(b"{social}.friends")); - assert!(graph_has_hash_tag(b"{partition}")); - assert!(!graph_has_hash_tag(b"social_graph")); - assert!(!graph_has_hash_tag(b"{}empty_tag")); - } - - #[test] - fn test_handle_traverse_graph_not_found() { - let store = GraphStore::new(); - let resp = handle_graph_traverse(&store, b"nonexistent", &[1, 2], None, u64::MAX - 1); - assert!(matches!(resp, Frame::Error(_))); - } - - #[test] - fn test_handle_traverse_empty_nodes() { - let mut store = GraphStore::new(); - store - .create_graph(Bytes::from_static(b"g"), 64_000, 1) - .expect("ok"); - - let resp = handle_graph_traverse(&store, b"g", &[], None, u64::MAX - 1); - if let Frame::Array(items) = &resp { - // Should have NEIGHBORS marker, empty array, DISCOVERED marker, empty array. - assert_eq!(items.len(), 4); - if let Frame::Array(neighbors) = &items[1] { - assert_eq!(neighbors.len(), 0); - } - if let Frame::Array(discovered) = &items[3] { - assert_eq!(discovered.len(), 0); - } - } else { - panic!("expected Array, got {:?}", resp); - } - } - - #[test] - fn test_handle_traverse_with_nodes() { - let mut store = GraphStore::new(); - store - .create_graph(Bytes::from_static(b"g"), 64_000, 1) - .expect("ok"); - - // Add nodes and edges via command dispatch. - let n1 = crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[b"GRAPH.ADDNODE", b"g", b"Person"]), - ); - let n2 = crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[b"GRAPH.ADDNODE", b"g", b"Person"]), - ); - - let id1 = if let Frame::Integer(id) = n1 { - id as u64 - } else { - panic!("expected int") - }; - let id2 = if let Frame::Integer(id) = n2 { - id as u64 - } else { - panic!("expected int") - }; - - let id1_str = id1.to_string(); - let id2_str = id2.to_string(); - crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[ - b"GRAPH.ADDEDGE", - b"g", - id1_str.as_bytes(), - id2_str.as_bytes(), - b"KNOWS", - ]), - ); - - // Traverse from node 1: should find node 2. - let resp = handle_graph_traverse(&store, b"g", &[id1], None, u64::MAX - 1); - if let Frame::Array(items) = &resp { - assert!(items.len() >= 4); - // NEIGHBORS should have frames (edge + node = 2 frames). - if let Frame::Array(neighbors) = &items[1] { - assert_eq!(neighbors.len(), 2); - } else { - panic!("expected Array for neighbors"); - } - // DISCOVERED should have 1 ID. - if let Frame::Array(discovered) = &items[3] { - assert_eq!(discovered.len(), 1); - if let Frame::Integer(disc_id) = &discovered[0] { - assert_eq!(*disc_id as u64, id2); - } - } else { - panic!("expected Array for discovered"); - } - } else { - panic!("expected Array, got {:?}", resp); - } - } - - #[test] - fn test_parse_traverse_response() { - // Build a mock response. - let response = Frame::Array( - vec![ - Frame::BulkString(Bytes::from_static(b"NEIGHBORS")), - Frame::Array(vec![Frame::Integer(42)].into()), - Frame::BulkString(Bytes::from_static(b"DISCOVERED")), - Frame::Array(vec![Frame::Integer(100), Frame::Integer(200)].into()), - ] - .into(), - ); - - let result = parse_traverse_response(&response); - assert!(result.is_some()); - let result = result.expect("should parse"); - assert_eq!(result.neighbor_ids, vec![100, 200]); - assert_eq!(result.frames.len(), 1); - assert!(!result.truncated); - } - - #[test] - fn test_parse_traverse_response_truncated() { - let response = Frame::Array( - vec![ - Frame::BulkString(Bytes::from_static(b"NEIGHBORS")), - Frame::Array(vec![].into()), - Frame::BulkString(Bytes::from_static(b"DISCOVERED")), - Frame::Array(vec![].into()), - Frame::BulkString(Bytes::from_static(b"TRUNCATED")), - ] - .into(), - ); - - let result = parse_traverse_response(&response).expect("should parse"); - assert!(result.truncated); - } - - #[test] - fn test_parse_traverse_response_invalid() { - assert!(parse_traverse_response(&Frame::Null).is_none()); - assert!(parse_traverse_response(&Frame::Integer(42)).is_none()); - assert!(parse_traverse_response(&Frame::Array(vec![Frame::Integer(1)].into())).is_none()); - } - - #[test] - fn test_default_depth_limit() { - assert_eq!(DEFAULT_CROSS_SHARD_DEPTH_LIMIT, 2); - } - - #[test] - fn test_handle_traverse_edge_type_filter() { - let mut store = GraphStore::new(); - store - .create_graph(Bytes::from_static(b"g"), 64_000, 1) - .expect("ok"); - - let n1 = crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[b"GRAPH.ADDNODE", b"g", b"Person"]), - ); - let n2 = crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[b"GRAPH.ADDNODE", b"g", b"Person"]), - ); - - let id1 = if let Frame::Integer(id) = n1 { - id as u64 - } else { - panic!("expected int") - }; - let id2 = if let Frame::Integer(id) = n2 { - id as u64 - } else { - panic!("expected int") - }; - - let id1_str = id1.to_string(); - let id2_str = id2.to_string(); - crate::command::graph::dispatch_graph_command( - &mut store, - &make_cmd(&[ - b"GRAPH.ADDEDGE", - b"g", - id1_str.as_bytes(), - id2_str.as_bytes(), - b"KNOWS", - ]), - ); - - // Filter by a type that doesn't match: should get empty results. - let resp = handle_graph_traverse(&store, b"g", &[id1], Some(9999), u64::MAX - 1); - if let Frame::Array(items) = &resp { - if let Frame::Array(neighbors) = &items[1] { - assert_eq!( - neighbors.len(), - 0, - "non-matching filter should yield no neighbors" - ); - } - } else { - panic!("expected Array"); - } - } -} diff --git a/src/graph/csr/mmap.rs b/src/graph/csr/mmap.rs index 86895149f..d5e722260 100644 --- a/src/graph/csr/mmap.rs +++ b/src/graph/csr/mmap.rs @@ -57,6 +57,12 @@ pub struct MmapCsrSegment { /// Lazily-built HNSW bridge over this segment's v5 embeddings (hybrid /// HnswPreFilter). DERIVED, in-memory only — never persisted. pub hnsw_bridge: std::sync::OnceLock>, + /// True while a background thread is building `hnsw_bridge` (W2-5). + pub hnsw_building: std::sync::atomic::AtomicBool, + /// Lazily built per-segment text index (`SegmentTextIndex`, P3 design + /// part B). DERIVED, in-memory only — never persisted, same as + /// `props_index`/`hnsw_bridge`. + pub text_index: std::sync::OnceLock, /// Pointer into mmap: node property blob (version >= 5; dangling+0 otherwise). node_props_ptr: *const u8, node_props_len: usize, @@ -370,6 +376,8 @@ impl MmapCsrSegment { incoming: std::sync::OnceLock::new(), props_index: std::sync::OnceLock::new(), hnsw_bridge: std::sync::OnceLock::new(), + hnsw_building: std::sync::atomic::AtomicBool::new(false), + text_index: std::sync::OnceLock::new(), node_props_ptr: np_ptr, node_props_len: np_len, edge_props_ptr: ep_ptr, diff --git a/src/graph/csr/mod.rs b/src/graph/csr/mod.rs index 09860e408..0f861ea6b 100644 --- a/src/graph/csr/mod.rs +++ b/src/graph/csr/mod.rs @@ -92,6 +92,16 @@ pub struct CsrSegment { /// HnswPreFilter). DERIVED, in-memory only — never persisted. `None` /// cached when the segment holds too few embeddings to earn one. pub hnsw_bridge: std::sync::OnceLock>, + /// True while a background thread is building `hnsw_bridge` (W2-5): the + /// build runs OFF the shard event loop; queries score exactly until the + /// bridge installs. + pub hnsw_building: std::sync::atomic::AtomicBool, + /// Lazily built per-segment text index for Cypher text predicates + /// (`SegmentTextIndex`, P3 design part B). DERIVED, in-memory only — + /// never persisted; a `GraphUnion`-merged segment starts with an empty + /// cell and rebuilds from scratch on first use (see `text_index.rs` + /// module docs, "GraphUnion merge" section). + pub text_index: std::sync::OnceLock, } impl CsrSegment { @@ -273,6 +283,8 @@ impl CsrSegment { incoming: std::sync::OnceLock::new(), props_index: std::sync::OnceLock::new(), hnsw_bridge: std::sync::OnceLock::new(), + hnsw_building: std::sync::atomic::AtomicBool::new(false), + text_index: std::sync::OnceLock::new(), }) } @@ -889,6 +901,8 @@ impl CsrSegment { incoming: std::sync::OnceLock::new(), props_index: std::sync::OnceLock::new(), hnsw_bridge: std::sync::OnceLock::new(), + hnsw_building: std::sync::atomic::AtomicBool::new(false), + text_index: std::sync::OnceLock::new(), }) } diff --git a/src/graph/csr/storage.rs b/src/graph/csr/storage.rs index ae01d8b41..748bbef91 100644 --- a/src/graph/csr/storage.rs +++ b/src/graph/csr/storage.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::path::Path; +use std::sync::Arc; use roaring::RoaringBitmap; use smallvec::SmallVec; @@ -166,7 +167,11 @@ impl CsrStorage { let hnsw = self .hnsw_bridge_if_built() .map_or(0, |b| b.resident_bytes()); - core + props + indexes + hnsw + let text = self.text_index_if_built().map_or( + 0, + crate::graph::text_index::SegmentTextIndex::resident_bytes, + ); + core + props + indexes + hnsw + text } /// Borrow the property index ONLY if it has already been built (does @@ -179,6 +184,16 @@ impl CsrStorage { } } + /// Borrow the text index ONLY if it has already been built (does not + /// trigger a build) — same `resident_bytes`-must-be-cheap rule as + /// `props_index_if_built`. + fn text_index_if_built(&self) -> Option<&crate::graph::text_index::SegmentTextIndex> { + match self { + CsrStorage::Heap(s) => s.text_index.get(), + CsrStorage::Mmap(s) => s.text_index.get(), + } + } + /// Borrow the HNSW bridge ONLY if it has already been built AND /// succeeded (`Some(GraphHnsw)`, not the cached "too few vectors" `None`). fn hnsw_bridge_if_built(&self) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { @@ -368,23 +383,64 @@ impl CsrStorage { } } - /// Get-or-build this segment's HNSW bridge over v5 node embeddings - /// (hybrid HnswPreFilter). Built at most once per segment, and only when - /// it holds >= `BRIDGE_MIN_VECTORS` usable embeddings — the negative - /// answer is cached too, so small segments pay the scan exactly once. - /// Same `OnceLock` interior-mutability pattern as `incoming_index`. - pub fn hnsw_bridge(&self) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { - let cell = match self { - CsrStorage::Heap(s) => &s.hnsw_bridge, - CsrStorage::Mmap(s) => &s.hnsw_bridge, + /// Get-or-build this segment's lazy text index over CSR rows (P3 design + /// part B). Built once from node_meta + the v5 property blob; cached in + /// the segment's `OnceLock` (same interior-mutability pattern as + /// `property_index`/`incoming_index`). + pub fn text_index(&self) -> &crate::graph::text_index::SegmentTextIndex { + match self { + CsrStorage::Heap(s) => s.text_index.get_or_init(|| { + crate::graph::text_index::SegmentTextIndex::build(&s.node_meta, &s.node_props) + }), + CsrStorage::Mmap(s) => s.text_index.get_or_init(|| { + crate::graph::text_index::SegmentTextIndex::build( + s.node_meta(), + s.node_props_blob(), + ) + }), + } + } + + /// This segment's HNSW bridge over v5 node embeddings (hybrid + /// HnswPreFilter), if already built. The build itself runs on a + /// detached background thread (W2-5): the first caller kicks it off and + /// gets `None` — every caller already treats `None` as "score exactly", + /// so queries brute-force until the bridge installs instead of stalling + /// the shard event loop for the full HNSW construction. The negative + /// answer (too few embeddings) is cached the same way. Built at most + /// once per segment. + pub fn hnsw_bridge(self: &Arc) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { + let (cell, building) = match &**self { + CsrStorage::Heap(s) => (&s.hnsw_bridge, &s.hnsw_building), + CsrStorage::Mmap(s) => (&s.hnsw_bridge, &s.hnsw_building), }; - cell.get_or_init(|| { - crate::graph::hnsw_bridge::GraphHnsw::build( - self, - crate::graph::hnsw_bridge::BRIDGE_MIN_VECTORS, - ) - }) - .as_ref() + if let Some(cached) = cell.get() { + return cached.as_ref(); + } + if !building.swap(true, std::sync::atomic::Ordering::AcqRel) { + let seg = Arc::clone(self); + std::thread::spawn(move || { + // A panicking build must still converge the cell (to None), + // or the segment would re-arm `building` on no future call + // and answer exact-only forever without a cached decision. + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + crate::graph::hnsw_bridge::GraphHnsw::build( + &seg, + crate::graph::hnsw_bridge::BRIDGE_MIN_VECTORS, + ) + })) + .unwrap_or_else(|_| { + tracing::warn!("graph hnsw bridge build panicked; segment stays exact-scored"); + None + }); + let cell = match &*seg { + CsrStorage::Heap(s) => &s.hnsw_bridge, + CsrStorage::Mmap(s) => &s.hnsw_bridge, + }; + let _ = cell.set(built); + }); + } + None } /// Test hook: build the bridge regardless of the production minimum, so @@ -616,6 +672,65 @@ mod tests { ); } + /// A heap-backed segment with real per-node STRING properties, so + /// `text_index()` has something non-trivial to build (mirrors the + /// numeric-property fixture's rationale above -- an all-numeric fixture + /// would build an empty text index and mask a broken resident_bytes + /// count with a true-by-accident 0 -> 0 "no growth" result). + fn heap_segment_with_string_props(n: usize) -> CsrStorage { + let mut g = MemGraph::new(1_000_000); + let mut keys = Vec::with_capacity(n); + for i in 0..n { + let props = smallvec![( + 0u16, + PropertyValue::String(bytes::Bytes::from(format!( + "the quick brown fox jumps over lazy dog number {i}" + ))), + )]; + keys.push(g.add_node(smallvec![0u16], props, None, 1)); + } + for i in 0..n.saturating_sub(1) { + g.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + let frozen = g.freeze().expect("freeze"); + let seg = CsrSegment::from_frozen(frozen, 10).expect("csr"); + CsrStorage::Heap(seg) + } + + /// P3 design part B, B1 test #3: `SegmentTextIndex` is absent until the + /// first `text_index()` call (mirrors `SegmentPropertyIndexes`'s lazy + /// build contract). + #[test] + fn test_text_index_lazy_build_on_first_use() { + let storage = heap_segment_with_string_props(8); + let baseline = storage.resident_bytes(); + let idx = storage.text_index(); + assert!( + !idx.is_empty(), + "fixture has a string property on every row" + ); + let after = storage.resident_bytes(); + assert!( + after > baseline, + "resident_bytes must grow once the lazily-built SegmentTextIndex is \ + materialized: baseline={baseline}, after={after}" + ); + } + + /// P3 design part B, B1 test #8: `resident_bytes` must be visible to the + /// shard's elastic memory budget the same way `SegmentPropertyIndexes` + /// and `GraphHnsw` already are (the exact bug class `CsrStorage:: + /// resident_bytes`'s doc comment records as previously fixed). + #[test] + fn test_text_index_resident_bytes_accounted() { + let storage = heap_segment_with_string_props(32); + let before = storage.resident_bytes(); + let _ = storage.text_index(); + let after = storage.resident_bytes(); + assert!(after > before, "before={before} after={after}"); + } + #[test] fn test_resident_bytes_mmap_zeroes_core_arrays_heap_counts_them() { // Heap segment: row/col/edge_meta/node_meta count in full (these are diff --git a/src/graph/cypher/ast.rs b/src/graph/cypher/ast.rs index 4b9da9217..594a7d549 100644 --- a/src/graph/cypher/ast.rs +++ b/src/graph/cypher/ast.rs @@ -300,6 +300,13 @@ pub enum BinaryOperator { Div, Mod, RegexMatch, + /// `CONTAINS` (P3 design part B): substring predicate, `n.prop CONTAINS + /// 'x'`. Desugars to the same evaluation `=~ ".*x.*"` already used. + Contains, + /// `STARTS WITH` (P3 design part B): prefix predicate. + StartsWith, + /// `ENDS WITH` (P3 design part B): suffix predicate. + EndsWith, } impl CypherQuery { diff --git a/src/graph/cypher/executor/eval.rs b/src/graph/cypher/executor/eval.rs index 76a63233c..fe0a1e46d 100644 --- a/src/graph/cypher/executor/eval.rs +++ b/src/graph/cypher/executor/eval.rs @@ -23,7 +23,7 @@ pub(crate) fn eval_expr( match expr { Expr::Integer(n) => Value::Int(*n), Expr::Float(f) => Value::Float(*f), - Expr::StringLit(s) => Value::String(s.clone()), + Expr::StringLit(s) => Value::String(Bytes::copy_from_slice(s.as_bytes())), Expr::Bool(b) => Value::Bool(*b), Expr::Null => Value::Null, @@ -283,7 +283,10 @@ pub(crate) fn eval_expr( match v { Value::Int(n) => Value::Int(n), Value::Float(f) => Value::Int(f as i64), - Value::String(s) => s.parse::().map_or(Value::Null, Value::Int), + Value::String(s) => core::str::from_utf8(&s) + .ok() + .and_then(|t| t.parse::().ok()) + .map_or(Value::Null, Value::Int), _ => Value::Null, } } else { @@ -304,7 +307,10 @@ pub(crate) fn eval_expr( match v { Value::Float(f) => Value::Float(f), Value::Int(n) => Value::Float(n as f64), - Value::String(s) => s.parse::().map_or(Value::Null, Value::Float), + Value::String(s) => core::str::from_utf8(&s) + .ok() + .and_then(|t| t.parse::().ok()) + .map_or(Value::Null, Value::Float), _ => Value::Null, } } else { @@ -322,7 +328,7 @@ pub(crate) fn eval_expr( snapshot_lsn, decay, ); - Value::String(value_to_string(&v)) + Value::String(Bytes::from(value_to_string(&v))) } else { Value::Null } @@ -476,20 +482,25 @@ pub(crate) fn eval_binary_op(left: &Value, op: BinaryOperator, right: &Value) -> BinaryOperator::NotEqual => { Value::Bool(compare_values(left, right) != std::cmp::Ordering::Equal) } - BinaryOperator::LessThan => { - Value::Bool(compare_values(left, right) == std::cmp::Ordering::Less) - } - BinaryOperator::GreaterThan => { - Value::Bool(compare_values(left, right) == std::cmp::Ordering::Greater) - } - BinaryOperator::LessEqual => { - let ord = compare_values(left, right); - Value::Bool(ord == std::cmp::Ordering::Less || ord == std::cmp::Ordering::Equal) - } - BinaryOperator::GreaterEqual => { - let ord = compare_values(left, right); - Value::Bool(ord == std::cmp::Ordering::Greater || ord == std::cmp::Ordering::Equal) - } + // Ordering comparisons follow openCypher: only same-kind operands + // are comparable (Int/Float promote to one numeric space); a + // cross-type, Null, or NaN comparison yields Null, which WHERE + // drops. This is ALSO what makes numeric index-range pruning sound + // (W2-3): the index excludes exactly the rows whose residual + // comparison is non-true. `compare_values` keeps its total + // type-rank order for ORDER BY / equality. + BinaryOperator::LessThan + | BinaryOperator::GreaterThan + | BinaryOperator::LessEqual + | BinaryOperator::GreaterEqual => match order_values(left, right) { + Some(ord) => Value::Bool(match op { + BinaryOperator::LessThan => ord == std::cmp::Ordering::Less, + BinaryOperator::GreaterThan => ord == std::cmp::Ordering::Greater, + BinaryOperator::LessEqual => ord != std::cmp::Ordering::Greater, + _ => ord != std::cmp::Ordering::Less, + }), + None => Value::Null, + }, // Arithmetic operators BinaryOperator::Add => match (left, right) { @@ -498,9 +509,10 @@ pub(crate) fn eval_binary_op(left: &Value, op: BinaryOperator, right: &Value) -> (Value::Int(a), Value::Float(b)) => Value::Float(*a as f64 + b), (Value::Float(a), Value::Int(b)) => Value::Float(a + *b as f64), (Value::String(a), Value::String(b)) => { - let mut s = a.clone(); - s.push_str(b); - Value::String(s) + let mut s = Vec::with_capacity(a.len() + b.len()); + s.extend_from_slice(a); + s.extend_from_slice(b); + Value::String(Bytes::from(s)) } _ => Value::Null, }, @@ -540,13 +552,14 @@ pub(crate) fn eval_binary_op(left: &Value, op: BinaryOperator, right: &Value) -> // suffix (prefix*), and contains (*middle*). match (left, right) { (Value::String(text), Value::String(pattern)) => { - let matched = if let Some(stripped) = pattern.strip_prefix(".*") { - if let Some(middle) = stripped.strip_suffix(".*") { - text.contains(middle) + let (text, pattern) = (text.as_ref(), pattern.as_ref()); + let matched = if let Some(stripped) = pattern.strip_prefix(b".*") { + if let Some(middle) = stripped.strip_suffix(b".*") { + bytes_contains(text, middle) } else { text.ends_with(stripped) } - } else if let Some(stripped) = pattern.strip_suffix(".*") { + } else if let Some(stripped) = pattern.strip_suffix(b".*") { text.starts_with(stripped) } else { text == pattern @@ -556,6 +569,34 @@ pub(crate) fn eval_binary_op(left: &Value, op: BinaryOperator, right: &Value) -> _ => Value::Null, } } + + // P3 design part B (B0): first-class CONTAINS / STARTS WITH / ENDS + // WITH -- pure syntax sugar over the same byte-level checks `=~` + // already performs for its three recognized shapes (see + // `test_eval_contains_matches_regex_dotstar_equivalent`). A non- + // String operand (including a missing property, which evaluates to + // Value::Null upstream) degrades to Value::Null, never a panic -- + // this is also the correctness anchor for the SegmentTextIndex + // SUPERSET contract: a row without a String value at the target + // property can never pass any of these three predicates. + BinaryOperator::Contains => match (left, right) { + (Value::String(text), Value::String(pattern)) => { + Value::Bool(bytes_contains(text.as_ref(), pattern.as_ref())) + } + _ => Value::Null, + }, + BinaryOperator::StartsWith => match (left, right) { + (Value::String(text), Value::String(pattern)) => { + Value::Bool(text.as_ref().starts_with(pattern.as_ref())) + } + _ => Value::Null, + }, + BinaryOperator::EndsWith => match (left, right) { + (Value::String(text), Value::String(pattern)) => { + Value::Bool(text.as_ref().ends_with(pattern.as_ref())) + } + _ => Value::Null, + }, } } @@ -563,6 +604,23 @@ pub(crate) fn eval_binary_op(left: &Value, op: BinaryOperator, right: &Value) -> // Value comparison (for Sort and equality) // --------------------------------------------------------------------------- +/// openCypher ordering comparability for `< > <= >=`: `Some(ord)` only when +/// both operands are the same kind (Int/Float promote to one numeric space, +/// String vs String, Bool vs Bool). Cross-type, Null, and NaN comparisons +/// return `None` (the operator yields Null). Distinct from `compare_values`, +/// which imposes a TOTAL type-rank order for ORDER BY. +fn order_values(a: &Value, b: &Value) -> Option { + match (a, b) { + (Value::Int(a), Value::Int(b)) => Some(a.cmp(b)), + (Value::Float(a), Value::Float(b)) => a.partial_cmp(b), + (Value::Int(a), Value::Float(b)) => (*a as f64).partial_cmp(b), + (Value::Float(a), Value::Int(b)) => a.partial_cmp(&(*b as f64)), + (Value::String(a), Value::String(b)) => Some(a.cmp(b)), + (Value::Bool(a), Value::Bool(b)) => Some(a.cmp(b)), + _ => None, + } +} + /// Compare two Values for ordering. /// NULL < Bool < Int/Float < String < Node < Edge < List < Map. pub(crate) fn compare_values(a: &Value, b: &Value) -> std::cmp::Ordering { @@ -607,14 +665,21 @@ pub(crate) fn compare_values(a: &Value, b: &Value) -> std::cmp::Ordering { // Helpers // --------------------------------------------------------------------------- -/// Convert a PropertyValue to a runtime Value. +/// Substring search over raw bytes (the `=~ ".*x.*"` fallback matcher). +fn bytes_contains(haystack: &[u8], needle: &[u8]) -> bool { + needle.is_empty() || haystack.windows(needle.len()).any(|w| w == needle) +} + +/// Convert a PropertyValue to a runtime Value. Zero-copy for strings/bytes +/// (refcount bump on the stored `Bytes`) — and binary-safe: pre-W2-4 this +/// lossily degraded non-UTF8 payloads to `""`. pub(crate) fn property_value_to_value(pv: &PropertyValue) -> Value { match pv { PropertyValue::Int(n) => Value::Int(*n), PropertyValue::Float(f) => Value::Float(*f), - PropertyValue::String(s) => Value::String(core::str::from_utf8(s).unwrap_or("").to_owned()), + PropertyValue::String(s) => Value::String(s.clone()), PropertyValue::Bool(b) => Value::Bool(*b), - PropertyValue::Bytes(b) => Value::String(core::str::from_utf8(b).unwrap_or("").to_owned()), + PropertyValue::Bytes(b) => Value::String(b.clone()), } } @@ -624,7 +689,7 @@ pub(crate) fn value_to_string(v: &Value) -> String { Value::Null => "null".into(), Value::Int(n) => n.to_string(), Value::Float(f) => f.to_string(), - Value::String(s) => s.clone(), + Value::String(s) => String::from_utf8_lossy(s).into_owned(), Value::Bool(b) => b.to_string(), Value::Node(k) => format!("node:{}", k.data().as_ffi()), Value::Edge(k) => format!("edge:{}", k.data().as_ffi()), @@ -691,7 +756,7 @@ pub(crate) fn value_to_property_value(v: &Value) -> Option { match v { Value::Int(n) => Some(PropertyValue::Int(*n)), Value::Float(f) => Some(PropertyValue::Float(*f)), - Value::String(s) => Some(PropertyValue::String(Bytes::from(s.clone()))), + Value::String(s) => Some(PropertyValue::String(s.clone())), Value::Bool(b) => Some(PropertyValue::Bool(*b)), // Path is not storable as a property value (CYP-05 runtime-only). Value::Path(_) => None, diff --git a/src/graph/cypher/executor/mod.rs b/src/graph/cypher/executor/mod.rs index 2a1df67e5..56abf43cc 100644 --- a/src/graph/cypher/executor/mod.rs +++ b/src/graph/cypher/executor/mod.rs @@ -31,7 +31,11 @@ pub enum Value { Null, Int(i64), Float(f64), - String(String), + /// Cypher string values are RESP bulk strings — arbitrary bytes, held as + /// `Bytes` so a stored property flows to the reply frame without copying + /// (W2-4) and non-UTF8 payloads survive the round-trip. Text ops + /// (`toInteger`, `=~`) validate UTF-8 at their own boundary. + String(Bytes), Bool(bool), Node(NodeKey), Edge(EdgeKey), @@ -93,9 +97,21 @@ impl SlotTable { } } PhysicalOp::Merge { pattern, .. } => table.bind_pattern(pattern), + // W2-13: a rebinding projection (WITH) re-seeds the row + // stream with its output names; RETURN (rebind: false) + // still binds nothing. + PhysicalOp::Project { items, rebind, .. } => { + if *rebind { + for item in items { + match &item.alias { + Some(alias) => table.bind(alias), + None => table.bind(&eval::expr_to_string(&item.expr)), + } + } + } + } // Reference-only operators bind nothing. PhysicalOp::Filter { .. } - | PhysicalOp::Project { .. } | PhysicalOp::Sort { .. } | PhysicalOp::Limit { .. } | PhysicalOp::Skip { .. } @@ -201,6 +217,24 @@ impl<'a> Row<'a> { } } +/// W2-13 OPTIONAL MATCH: emit `row` with the expansion's target (and edge +/// variable, when the pattern binds one) set to Null — the survival row for +/// a source that matched nothing. Free function (not a closure) so the +/// `Row<'a>` lifetime unifies between the borrowed input and the output Vec. +fn push_null_padded<'a>( + row: &Row<'a>, + target: &str, + edge_variable: &Option, + out: &mut Vec>, +) { + let mut new_row = row.clone(); + new_row.insert(target, Value::Null); + if let Some(evar) = edge_variable { + new_row.insert(evar, Value::Null); + } + out.push(new_row); +} + /// Execution error. /// /// Phase 174 FIX-02: carries `partial_mutations` so that even on Err, any @@ -272,13 +306,20 @@ pub enum MutationRecord { // --- Phase 174 FIX-01: SET / DELETE / MERGE rollback records --- /// Property was changed by Cypher SET. `old_value` is the pre-SET value /// (None = property did not exist before SET and should be removed on - /// rollback). + /// rollback). `new_value` is the value written — serialized to the WAL + /// (W2-9: without it, SET was silently lost on kill -9 because replay + /// only re-ran the original ADDNODE property state). SetProperty { entity_id: u64, is_node: bool, key: u16, old_value: Option, + new_value: PropertyValue, }, + /// Label was added by Cypher `SET n:Label` (W2-9: WAL durability; label + /// rollback was never captured — pre-existing Phase 174 scope — so this + /// record produces a WAL entry but no undo op). + SetLabel { node_id: u64, label: u16 }, /// Node was soft-deleted by Cypher DETACH DELETE. Snapshot captures the /// full node state so rollback can un-soft-delete the node and its /// incident edges. @@ -671,6 +712,67 @@ mod tests { )); } + /// P3 design part B (B0): CONTAINS / STARTS WITH / ENDS WITH must + /// produce results IDENTICAL to the equivalent `=~` dot-star shape + /// already supported (`eval.rs` "three recognized shapes"). + #[test] + fn test_eval_contains_matches_regex_dotstar_equivalent() { + let text = Value::String("trusted rustacean".into()); + let contains = eval_binary_op( + &text, + BinaryOperator::Contains, + &Value::String("rust".into()), + ); + let regex_equiv = eval_binary_op( + &text, + BinaryOperator::RegexMatch, + &Value::String(".*rust.*".into()), + ); + assert!(matches!(contains, Value::Bool(true))); + assert!(matches!(regex_equiv, Value::Bool(true))); + + let starts = eval_binary_op( + &text, + BinaryOperator::StartsWith, + &Value::String("trust".into()), + ); + let starts_regex = eval_binary_op( + &text, + BinaryOperator::RegexMatch, + &Value::String("trust.*".into()), + ); + assert!(matches!(starts, Value::Bool(true))); + assert!(matches!(starts_regex, Value::Bool(true))); + + let ends = eval_binary_op( + &text, + BinaryOperator::EndsWith, + &Value::String("rustacean".into()), + ); + let ends_regex = eval_binary_op( + &text, + BinaryOperator::RegexMatch, + &Value::String(".*rustacean".into()), + ); + assert!(matches!(ends, Value::Bool(true))); + assert!(matches!(ends_regex, Value::Bool(true))); + + // Non-string operands and non-matches degrade to Null / false, same + // as `=~` already does -- never a wrong-type panic. + assert!(matches!( + eval_binary_op(&Value::Int(1), BinaryOperator::Contains, &text), + Value::Null + )); + assert!(matches!( + eval_binary_op( + &text, + BinaryOperator::StartsWith, + &Value::String("zzz".into()) + ), + Value::Bool(false) + )); + } + #[test] fn test_execute_merge_create_when_not_found() { let mut store = GraphStore::new(); @@ -775,4 +877,365 @@ mod tests { assert_eq!(graph_ref.write_buf.node_count(), 2); assert_eq!(graph_ref.write_buf.edge_count(), 1); } + + // --- Mutable-tier property index (Task #31) — executor-level correctness --- + + fn run_point_match(store: &GraphStore, target: i64) -> ExecResult { + let query = format!("MATCH (a:N {{id: {target}}}) RETURN a.id"); + let parsed = crate::graph::cypher::parse_cypher(query.as_bytes()).expect("parse"); + let plan = crate::graph::cypher::planner::compile(&parsed).expect("compile"); + let graph = store.get_graph(b"test").expect("graph"); + execute(graph, &plan, &HashMap::new(), &ExecutionContext::default()).expect("exec") + } + + /// Proves `MATCH (a:N {id:X})` returns exactly the one matching node at + /// several graph sizes — pure correctness regression guard for the + /// mutable-tier property index replacing the O(N) linear scan. + #[test] + fn test_index_scan_correctness_at_scale() { + for n in [1usize, 100, 5_000] { + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"test"), n * 2 + 1, 0) + .expect("create ok"); + let graph_mut = store.get_graph_mut(b"test").expect("graph"); + let label_id = label_to_id(b"N"); + let id_pid = label_to_id(b"id"); + for i in 0..n { + let mut props = SmallVec::new(); + props.push((id_pid, PropertyValue::Int(i as i64))); + graph_mut + .write_buf + .add_node(SmallVec::from_elem(label_id, 1), props, None, 1); + } + + let target = (n as i64) / 2; // arbitrary in-range id + + // Probe hook proving the linear scan is gone: `index_scan_keys` + // seeds its mutable-tail candidate set from EXACTLY this same + // accessor (`prop_index_keys_eq`). A bucket cardinality of 1 + // regardless of `n` is the structural proof that the seeded + // candidate set — and therefore the executor's residual-check + // loop — is O(bucket size), not O(live_node_count). + let candidate_count = graph_mut + .write_buf + .prop_index_keys_eq(id_pid, &PropertyValue::Int(target)) + .len(); + assert_eq!( + candidate_count, 1, + "n={n}: index probe must return exactly the matching bucket, \ + not scale with graph size" + ); + + let result = run_point_match(&store, target); + assert_eq!(result.rows.len(), 1, "n={n}: expected exactly one match"); + match &result.rows[0][0] { + Value::Int(v) => assert_eq!(*v, target, "n={n}: wrong node matched"), + other => panic!("n={n}: expected Int, got {other:?}"), + } + } + } + + /// After `SET n.id = newval`, the OLD value must no longer be + /// index-reachable and the NEW value must be. Guards the + /// `set_node_property` old-bucket-eviction path. + #[test] + fn test_index_scan_eq_after_property_update() { + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"test"), 64_000, 0) + .expect("create ok"); + let graph_mut = store.get_graph_mut(b"test").expect("graph"); + let label_id = label_to_id(b"N"); + let id_pid = label_to_id(b"id"); + let mut props = SmallVec::new(); + props.push((id_pid, PropertyValue::Int(1))); + graph_mut + .write_buf + .add_node(SmallVec::from_elem(label_id, 1), props, None, 1); + + let set_query = + crate::graph::cypher::parse_cypher(b"MATCH (n:N {id: 1}) SET n.id = 2 RETURN n") + .expect("parse"); + let set_plan = crate::graph::cypher::planner::compile(&set_query).expect("compile"); + let graph = store.get_graph_mut(b"test").expect("graph"); + let set_result = execute_mut(graph, &set_plan, &HashMap::new(), 0).expect("exec"); + assert_eq!(set_result.properties_set, 1); + + assert_eq!( + run_point_match(&store, 2).rows.len(), + 1, + "new value must match" + ); + assert_eq!( + run_point_match(&store, 1).rows.len(), + 0, + "old value must not match" + ); + } + + /// W2-2 copy-up interaction: freeze a node with `id: 1` into a CSR + /// segment, then SET it to `id: 2` (triggers copy-up into the mutable + /// tier). `MATCH {id: 1}` must return EMPTY (the resident-but-updated + /// mutable copy shadows the frozen row) and `MATCH {id: 2}` must find it + /// via the new mutable-tier index. + #[test] + fn test_index_scan_after_copy_up_shadows_frozen_value() { + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"test"), 1_000, 0) + .expect("create ok"); + let graph_mut = store.get_graph_mut(b"test").expect("graph"); + let label_id = label_to_id(b"N"); + let id_pid = label_to_id(b"id"); + let mut props = SmallVec::new(); + props.push((id_pid, PropertyValue::Int(1))); + graph_mut + .write_buf + .add_node(SmallVec::from_elem(label_id, 1), props, None, 1); + + // Freeze into a CSR segment so the row lives only in the frozen tier. + assert!(graph_mut.freeze_and_compact(1)); + assert_eq!(graph_mut.write_buf.node_count(), 0); + + // SET copies the frozen row up into the mutable tier, then mutates. + let set_query = + crate::graph::cypher::parse_cypher(b"MATCH (n:N {id: 1}) SET n.id = 2 RETURN n") + .expect("parse"); + let set_plan = crate::graph::cypher::planner::compile(&set_query).expect("compile"); + let graph = store.get_graph_mut(b"test").expect("graph"); + let set_result = execute_mut(graph, &set_plan, &HashMap::new(), 2).expect("exec"); + assert_eq!(set_result.properties_set, 1); + + assert_eq!( + run_point_match(&store, 1).rows.len(), + 0, + "frozen id=1 must be shadowed by the updated mutable copy" + ); + assert_eq!( + run_point_match(&store, 2).rows.len(), + 1, + "updated id=2 must be found via the mutable-tier index" + ); + } + + // --- P3 design part B: text predicates (CONTAINS/STARTS WITH/ENDS + // WITH/=~), SegmentTextIndex correctness across tiers ----------------- + + /// How the fixture's 7 nodes are distributed across tiers. + enum TierShape { + /// All 7 nodes stay in the mutable write buffer (no freeze). + MutableOnly, + /// All 7 nodes are frozen into one CSR segment. + FrozenOnly, + /// First 4 nodes frozen into a CSR segment; remaining 3 added to + /// the mutable tier AFTER the freeze (genuinely mixed tiers, not + /// just "freeze everything then leave it"). + Mixed, + } + + /// Fixture: (name, bio) pairs deliberately covering the edge cases the + /// SUPERSET-candidate contract must survive: + /// - "bob"/"trusted colleague": the substring-across-token-boundary + /// crux -- "trusted" tokenizes differently than "rust", so a + /// token-identity index would MISS it for `CONTAINS 'rust'`; the + /// presence-only `SegmentTextIndex` must not. + /// - "carol"/"RUSTACEAN": case-sensitivity -- present (has a string + /// bio) but must be excluded by the exact residual Filter, not by + /// the index (index has no opinion on case at all). + /// - "dave": NO bio property at all -- must never appear in any bio + /// predicate result, in any tier. + /// - "erin"/"" (empty bio): empty-string edge case. + /// - "frank"/"héllo wörld": Unicode multi-byte edge case. + /// - "grace"/"rust": exact single-token baseline. + const TEXT_FIXTURE: &[(&str, Option<&str>)] = &[ + ("alice", Some("i love rust and graphs")), + ("bob", Some("trusted colleague")), + ("carol", Some("RUSTACEAN")), + ("dave", None), + ("erin", Some("")), + ("frank", Some("héllo wörld")), + ("grace", Some("rust")), + ]; + + fn add_text_fixture_nodes( + graph: &mut crate::graph::store::NamedGraph, + entries: &[(&str, Option<&str>)], + lsn: u64, + ) { + let label_id = label_to_id(b"N"); + let name_pid = label_to_id(b"name"); + let bio_pid = label_to_id(b"bio"); + for (name, bio) in entries { + let mut props: SmallVec<[(u16, PropertyValue); 4]> = SmallVec::new(); + props.push(( + name_pid, + PropertyValue::String(Bytes::copy_from_slice(name.as_bytes())), + )); + if let Some(bio) = bio { + props.push(( + bio_pid, + PropertyValue::String(Bytes::copy_from_slice(bio.as_bytes())), + )); + } + graph + .write_buf + .add_node(SmallVec::from_elem(label_id, 1), props, None, lsn); + } + } + + fn build_text_fixture_store(shape: TierShape) -> GraphStore { + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"test"), 1_000_000, 0) + .expect("create ok"); + let graph = store.get_graph_mut(b"test").expect("graph"); + match shape { + TierShape::MutableOnly => { + add_text_fixture_nodes(graph, TEXT_FIXTURE, 1); + } + TierShape::FrozenOnly => { + add_text_fixture_nodes(graph, TEXT_FIXTURE, 1); + assert!(graph.freeze_and_compact(1), "freeze must succeed"); + assert_eq!(graph.write_buf.node_count(), 0); + } + TierShape::Mixed => { + add_text_fixture_nodes(graph, &TEXT_FIXTURE[..4], 1); + assert!(graph.freeze_and_compact(1), "freeze must succeed"); + add_text_fixture_nodes(graph, &TEXT_FIXTURE[4..], 2); + } + } + store + } + + /// Run a Cypher query returning `n.name` and collect the sorted set of + /// matched names. + fn run_text_query(store: &GraphStore, cypher: &str) -> Vec { + let parsed = crate::graph::cypher::parse_cypher(cypher.as_bytes()).expect("parse"); + let plan = crate::graph::cypher::planner::compile(&parsed).expect("compile"); + let graph = store.get_graph(b"test").expect("graph"); + let result = execute(graph, &plan, &HashMap::new(), &ExecutionContext::default()) + .unwrap_or_else(|e| panic!("exec failed for {cypher:?}: {e:?}")); + let mut names: Vec = result + .rows + .iter() + .map(|r| match &r[0] { + Value::String(s) => String::from_utf8_lossy(s).into_owned(), + other => panic!("expected string name, got {other:?}"), + }) + .collect(); + names.sort(); + names + } + + /// The single most important correctness gate for P3 design part B: + /// every text-predicate query must return the IDENTICAL result set + /// regardless of which tier(s) the matching data lives in. `IndexScan` + /// accelerates only the frozen tier (`SegmentTextIndex::candidate_rows`) + /// -- the mutable tier always falls back to an exact scan, and the + /// residual `Filter` is the sole authority either way, so tier + /// placement must be invisible to the result set. + #[test] + fn test_text_predicate_parity_across_tiers() { + let queries: &[(&str, &[&str])] = &[ + ( + "MATCH (n:N) WHERE n.bio CONTAINS 'rust' RETURN n.name", + &["alice", "bob", "grace"], + ), + ( + // Empty needle: matches every row that HAS a bio (bytes_contains + // treats "" as always-contained), but never `dave` (no bio at all). + "MATCH (n:N) WHERE n.bio CONTAINS '' RETURN n.name", + &["alice", "bob", "carol", "erin", "frank", "grace"], + ), + ( + "MATCH (n:N) WHERE n.bio STARTS WITH 'trust' RETURN n.name", + &["bob"], + ), + ( + // Unicode multi-byte suffix. + "MATCH (n:N) WHERE n.bio ENDS WITH 'ld' RETURN n.name", + &["frank"], + ), + ( + "MATCH (n:N) WHERE n.bio =~ '.*rust.*' RETURN n.name", + &["alice", "bob", "grace"], + ), + ( + // Case-sensitivity: 'RUSTACEAN' must NOT match lowercase 'rust' + // -- proves the residual Filter (not the presence-only index) + // makes the final call. + "MATCH (n:N) WHERE n.bio CONTAINS 'RUST' RETURN n.name", + &["carol"], + ), + ]; + + for shape in [ + TierShape::MutableOnly, + TierShape::FrozenOnly, + TierShape::Mixed, + ] { + let shape_name = match shape { + TierShape::MutableOnly => "MutableOnly", + TierShape::FrozenOnly => "FrozenOnly", + TierShape::Mixed => "Mixed", + }; + let store = build_text_fixture_store(shape); + for (cypher, expected) in queries { + let mut expected: Vec = expected.iter().map(|s| s.to_string()).collect(); + expected.sort(); + let actual = run_text_query(&store, cypher); + assert_eq!( + actual, expected, + "tier={shape_name} query={cypher:?}: got {actual:?}, want {expected:?}" + ); + } + } + } + + /// B1 test #5/#6: the presence-only `candidate_rows` superset must + /// survive a query that a naive token-identity index would get wrong. + /// `IndexScan` is planned (via `CONTAINS`) but the residual Filter + /// still produces the exact, correct row set -- this is the same + /// assertion as `test_text_predicate_parity_across_tiers`'s first case, + /// isolated here with an explicit plan-shape check. + #[test] + fn test_text_scan_superset_semantics_frozen_tier() { + let store = build_text_fixture_store(TierShape::FrozenOnly); + let query = "MATCH (n:N) WHERE n.bio CONTAINS 'rust' RETURN n.name"; + let parsed = crate::graph::cypher::parse_cypher(query.as_bytes()).expect("parse"); + let plan = crate::graph::cypher::planner::compile(&parsed).expect("compile"); + assert!( + matches!( + plan.operators[0], + crate::graph::cypher::planner::PhysicalOp::IndexScan { .. } + ), + "CONTAINS must plan through IndexScan; ops = {:?}", + plan.operators + ); + let mut actual = run_text_query(&store, query); + actual.sort(); + // "bob" ("trusted colleague") is the false-negative-if-token-pruned + // case; it MUST be present because the index only prunes on + // presence, never on token identity. + assert_eq!(actual, vec!["alice", "bob", "grace"]); + } + + /// B1 test #7: the mutable tier gets no acceleration for text + /// predicates (pre-approved scope decision) but must not regress + /// correctness -- covered by `test_text_predicate_parity_across_tiers`'s + /// `MutableOnly` shape; this test isolates the plan shape (still an + /// `IndexScan`, since the WHERE conjunct always upgrades the scan + /// regardless of tier -- the ACCELERATION difference is inside + /// `index_scan_keys`, not in the plan). + #[test] + fn test_text_scan_mutable_tier_falls_back_to_linear_scan() { + let store = build_text_fixture_store(TierShape::MutableOnly); + let mut actual = run_text_query( + &store, + "MATCH (n:N) WHERE n.bio CONTAINS 'rust' RETURN n.name", + ); + actual.sort(); + assert_eq!(actual, vec!["alice", "bob", "grace"]); + } } diff --git a/src/graph/cypher/executor/read.rs b/src/graph/cypher/executor/read.rs index 6d54d2d6b..ab573ab0b 100644 --- a/src/graph/cypher/executor/read.rs +++ b/src/graph/cypher/executor/read.rs @@ -2,6 +2,178 @@ use std::collections::HashMap; use super::*; +/// W2-12: if `expr` is a top-level aggregate call, return +/// `(lowercase name, input expr, distinct)`. `None` input = `count(*)` / +/// bare `count()` (counts rows, not values). Aggregates nested inside a +/// larger expression (`count(n) + 1`) are NOT recognized — the item then +/// evaluates per-row like any scalar (pre-existing behavior), so keep +/// aggregates at the top level of a RETURN item. +fn aggregate_call(expr: &Expr) -> Option<(&'static str, Option<&Expr>, bool)> { + let Expr::FunctionCall { + name, + args, + distinct, + } = expr + else { + return None; + }; + let canon: &'static str = match name.to_ascii_lowercase().as_str() { + "count" => "count", + "sum" => "sum", + "avg" => "avg", + "min" => "min", + "max" => "max", + "collect" => "collect", + _ => return None, + }; + let input = args.first().filter(|a| !matches!(a, Expr::Star)); + Some((canon, input, *distinct)) +} + +/// W2-12: aggregate projection with implicit grouping (openCypher): the +/// non-aggregate RETURN items form the group key; each group emits one row. +/// No group key + zero input rows still emits ONE row (`count` = 0, `sum` = +/// 0, `collect` = [], others Null); a present group key over zero rows +/// emits none. Null inputs are skipped by every aggregate except `count(*)`, +/// which counts rows. Returns `None` when no item is an aggregate (caller +/// takes the plain per-row projection path). +fn try_project_aggregate( + items: &[ReturnItem], + row_count: usize, + mut eval_at: impl FnMut(&Expr, usize) -> Value, +) -> Option>> { + let specs: Vec, bool)>> = + items.iter().map(|it| aggregate_call(&it.expr)).collect(); + if specs.iter().all(Option::is_none) { + return None; + } + let has_group_key = specs.iter().any(Option::is_none); + let agg_count = specs.iter().filter(|s| s.is_some()).count(); + + // Group rows by the stringified key (same "simple approach" as + // dedup_rows), keeping first-seen order for deterministic output. + struct Group { + key_values: Vec, + inputs: Vec>, + } + let mut order: Vec = Vec::new(); + let mut groups: HashMap = HashMap::new(); + + for ri in 0..row_count { + let mut key_values = Vec::new(); + let mut key_str = String::new(); + for (item, spec) in items.iter().zip(&specs) { + if spec.is_none() { + let v = eval_at(&item.expr, ri); + key_str.push_str(&value_to_string(&v)); + key_str.push('\u{1f}'); + key_values.push(v); + } + } + let group = groups.entry(key_str.clone()).or_insert_with(|| { + order.push(key_str); + Group { + key_values, + inputs: vec![Vec::new(); agg_count], + } + }); + for (ai, spec) in specs.iter().flatten().enumerate() { + match spec.1 { + Some(input_expr) => { + let v = eval_at(input_expr, ri); + if !matches!(v, Value::Null) { + group.inputs[ai].push(v); + } + } + // count(*): every row counts. + None => group.inputs[ai].push(Value::Int(1)), + } + } + } + + // Global aggregate over zero rows: one synthetic empty group. + if groups.is_empty() && !has_group_key { + let key = String::new(); + order.push(key.clone()); + groups.insert( + key, + Group { + key_values: Vec::new(), + inputs: vec![Vec::new(); agg_count], + }, + ); + } + + let finalize = |name: &str, mut inputs: Vec, distinct: bool| -> Value { + if distinct { + let mut seen = std::collections::HashSet::new(); + inputs.retain(|v| seen.insert(value_to_string(v))); + } + match name { + "count" => Value::Int(inputs.len() as i64), + "sum" => { + if inputs.iter().any(|v| matches!(v, Value::Float(_))) { + Value::Float(inputs.iter().fold(0.0, |acc, v| match v { + Value::Int(i) => acc + *i as f64, + Value::Float(f) => acc + f, + _ => acc, + })) + } else { + Value::Int(inputs.iter().fold(0i64, |acc, v| match v { + Value::Int(i) => acc.saturating_add(*i), + _ => acc, + })) + } + } + "avg" => { + let numeric: Vec = inputs + .iter() + .filter_map(|v| match v { + Value::Int(i) => Some(*i as f64), + Value::Float(f) => Some(*f), + _ => None, + }) + .collect(); + if numeric.is_empty() { + Value::Null + } else { + Value::Float(numeric.iter().sum::() / numeric.len() as f64) + } + } + "min" => inputs + .into_iter() + .min_by(compare_values) + .unwrap_or(Value::Null), + "max" => inputs + .into_iter() + .max_by(compare_values) + .unwrap_or(Value::Null), + "collect" => Value::List(inputs), + _ => Value::Null, + } + }; + + let mut out = Vec::with_capacity(order.len()); + for key in &order { + let Some(group) = groups.remove(key) else { + continue; + }; + let mut key_iter = group.key_values.into_iter(); + let mut input_iter = group.inputs.into_iter(); + let row: Vec = specs + .iter() + .map(|spec| match spec { + None => key_iter.next().unwrap_or(Value::Null), + Some((name, _, distinct)) => { + finalize(name, input_iter.next().unwrap_or_default(), *distinct) + } + }) + .collect(); + out.push(row); + } + Some(out) +} + /// Resolve an `IndexScan` into the matching node keys across both tiers. /// /// Frozen tier: per segment, intersect the property-equality bitmaps @@ -13,12 +185,34 @@ use super::*; /// `prop_eq` values are literals or parameters; if any resolves to a /// non-scalar the whole scan degrades to the plain merged label scan (the /// residual Filter downstream keeps results exact either way). +/// +/// `prop_range` conjuncts (`n.p > $x` etc.) prune via the per-segment +/// numeric B-trees. A conjunct whose threshold resolves non-numeric is +/// dropped (can't prune the numeric space) — never narrowed: the pruned set +/// stays a SUPERSET of the rows the residual Filter accepts, because the +/// post-W2-3 comparison semantics make cross-type / missing-property +/// comparisons evaluate to Null (dropped by WHERE), exactly the rows the +/// numeric index excludes. +/// +/// `text_pred` conjuncts (`n.p CONTAINS 'x'`, `STARTS WITH`, `ENDS WITH`, +/// `=~`; P3 design part B) prune the FROZEN tier only, via +/// `SegmentTextIndex::candidate_rows` — a PRESENCE-only superset (rows +/// whose value at that property is a String/Bytes at all; see +/// `text_index.rs` module docs for why token-level pruning is unsound for +/// substring/prefix/suffix predicates). The MUTABLE tier has no text index +/// (pre-approved scope decision): a text-only conjunct (no `prop_eq`/ +/// `prop_range` alongside it) falls back to an exact full scan of the +/// mutable tail, seeded from `memgraph.iter_nodes()` — correct, just +/// unaccelerated, matching the mutable tier's existing story for numeric +/// properties before freeze. #[allow(clippy::too_many_arguments)] fn index_scan_keys( memgraph: &crate::graph::memgraph::MemGraph, csr_segs: &[std::sync::Arc], label: Option<&String>, prop_eq: &[(String, Expr)], + prop_range: &[(String, RangeCmp, Expr)], + text_pred: &[(String, BinaryOperator, Expr)], params: &HashMap, ctx: &ExecutionContext, ) -> Vec { @@ -59,10 +253,102 @@ fn index_scan_keys( } } + // Resolve each range conjunct to (prop_id, cmp, numeric threshold). A + // threshold that resolves non-numeric (e.g. a String parameter) cannot + // prune the numeric index — drop the conjunct (superset-safe); the + // residual Filter stays exact. + let mut ranges: Vec<(u16, RangeCmp, f64)> = Vec::with_capacity(prop_range.len()); + for (name, cmp, expr) in prop_range { + let v = eval_expr( + expr, + &empty_row, + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ); + match v { + Value::Int(i) => ranges.push((label_to_id(name.as_bytes()), *cmp, i as f64)), + Value::Float(f) => ranges.push((label_to_id(name.as_bytes()), *cmp, f)), + _ => {} + } + } + + // Resolve each text conjunct to a bare prop_id (deduplicated). No value + // is evaluated: `SegmentTextIndex::candidate_rows` prunes on PRESENCE + // alone (see its module docs) -- valid as a superset regardless of the + // pattern's content, so the pattern expression is never consulted here. + let mut text_prop_ids: Vec = Vec::with_capacity(text_pred.len()); + for (name, _op, _expr) in text_pred { + let pid = label_to_id(name.as_bytes()); + if !text_prop_ids.contains(&pid) { + text_prop_ids.push(pid); + } + } + + // Nothing prunable (pure-range scan whose thresholds all resolved + // non-numeric, and no text conjunct either): full merged label scan, + // residual Filter stays exact. + if targets.is_empty() && ranges.is_empty() && text_prop_ids.is_empty() { + let mut keys = Vec::new(); + view.for_each_visible_node( + label_id, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + |k| keys.push(k), + ); + return keys; + } + let mut keys = Vec::new(); - // Mutable tail: inline superset-consistent property check. - for (key, node) in memgraph.iter_nodes() { + // Mutable tail: seed a small candidate set from the mutable-tier + // property index (Task #31) instead of scanning every live node, then + // apply the SAME superset-consistent label/visibility/property checks + // as before to that candidate set. SUPERSET contract preserved: the + // index only prunes candidates, never decides — these checks (and the + // planner's residual Filter downstream) stay authoritative. + // + // Seed from the first equality target when available (exact bucket, + // typically 0-1 entries for a unique id); otherwise from the full + // indexed value range of the first range conjunct (conservative — the + // `ranges_match` check below narrows it exactly). One of the two is + // always present here: `targets.is_empty() && ranges.is_empty()` was + // already handled by the early return above. + let candidates: Vec = if let Some((pid, want)) = targets.first() { + memgraph.prop_index_keys_eq(*pid, want).to_vec() + } else if let Some((pid, cmp, threshold)) = ranges.first() { + // Boundary-INCLUSIVE on the threshold side regardless of Gt/Lt vs + // Gte/Lte — a superset seed; the exact `ranges_match` check below + // enforces strictness. + let (lo, hi) = match cmp { + RangeCmp::Gt | RangeCmp::Gte => (*threshold, f64::INFINITY), + RangeCmp::Lt | RangeCmp::Lte => (f64::NEG_INFINITY, *threshold), + }; + memgraph.prop_index_keys_range(*pid, lo, hi) + } else if !text_prop_ids.is_empty() { + // Text-only conjunct(s): the mutable tier has no text index + // (pre-approved scope decision, see `text_index.rs` module docs) -- + // fall back to a full label-scoped scan of the mutable tail. Still + // bounded by `edge_threshold` (freeze keeps the mutable tail + // small), and the generic label/visibility checks below plus the + // planner's residual Filter downstream stay authoritative, exactly + // like every other candidate source in this function. + memgraph.iter_nodes().map(|(key, _)| key).collect() + } else { + // Structurally unreachable (see above) — an empty candidate set is + // the safe degradation if this invariant is ever violated by a + // future refactor, never a panic on live traffic. + Vec::new() + }; + + for key in candidates { + let Some(node) = memgraph.get_node(key) else { + continue; // tombstone/race guard: index entry outlived the node + }; if let Some(lid) = label_id { if !node.labels.contains(&lid) { continue; @@ -82,12 +368,31 @@ fn index_scan_keys( .iter() .any(|(id, have)| id == pid && prop_value_loose_eq(have, want)) }); - if all_match { + // Range check mirrors the index's numeric normalization (Int/Float/ + // Bool as 0-1 in one f64 space) so both tiers prune identically. + let ranges_match = ranges.iter().all(|(pid, cmp, threshold)| { + node.properties.iter().any(|(id, have)| { + if id != pid { + return false; + } + let num = match have { + PropertyValue::Int(i) => *i as f64, + PropertyValue::Float(f) => *f, + PropertyValue::Bool(b) => f64::from(u8::from(*b)), + _ => return false, + }; + range_cmp_holds(num, *cmp, *threshold) + }) + }); + if all_match && ranges_match { keys.push(key); } } - // Frozen tier: bitmap intersection per segment. + // Frozen tier: bitmap intersection per segment. `emitted` dedups keys a + // re-frozen copy-up shadow left in multiple segments (stale-index hits + // are dropped by the planner's residual Filter downstream). + let mut emitted = crate::graph::fasthash::FxHashSet::default(); for seg in csr_segs { let mut bm: Option = None; for (pid, pv) in &targets { @@ -102,6 +407,42 @@ fn index_scan_keys( } bm = Some(acc); } + for (pid, cmp, threshold) in &ranges { + if bm.as_ref().is_some_and(roaring::RoaringBitmap::is_empty) { + break; + } + let rows = match seg.property_index().numeric_index(*pid) { + Some(ix) => match cmp { + RangeCmp::Gt => ix.gt(*threshold), + RangeCmp::Gte => ix.gte(*threshold), + RangeCmp::Lt => ix.lt(*threshold), + RangeCmp::Lte => ix.lte(*threshold), + }, + // No numeric value ever indexed under this prop in this + // segment -> no row here can pass the residual comparison. + None => roaring::RoaringBitmap::new(), + }; + let acc = match bm.take() { + Some(acc) => acc & rows, + None => rows, + }; + bm = Some(acc); + } + for pid in &text_prop_ids { + if bm.as_ref().is_some_and(roaring::RoaringBitmap::is_empty) { + break; + } + let rows = seg + .text_index() + .candidate_rows(*pid) + .cloned() + .unwrap_or_default(); + let acc = match bm.take() { + Some(acc) => acc & rows, + None => rows, + }; + bm = Some(acc); + } let Some(mut bm) = bm else { continue }; if bm.is_empty() { continue; @@ -126,13 +467,33 @@ fn index_scan_keys( ) { continue; } - keys.push(NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id))); + let key = NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id)); + // Copy-up shadow (W2-2): the mutable tier overrides this row — + // a live shadow was already scanned above (with its CURRENT + // property values); a dead shadow is a tombstone. + if memgraph.get_node(key).is_some() { + continue; + } + if !emitted.insert(key) { + continue; + } + keys.push(key); } } keys } +/// Numeric range comparison for the mutable-tail index check. +fn range_cmp_holds(value: f64, cmp: RangeCmp, threshold: f64) -> bool { + match cmp { + RangeCmp::Gt => value > threshold, + RangeCmp::Gte => value >= threshold, + RangeCmp::Lt => value < threshold, + RangeCmp::Lte => value <= threshold, + } +} + /// Superset-consistent equality between stored and queried property values, /// mirroring the index's numeric normalization (Int/Float/Bool share one /// f64 space; String/Bytes compare bytewise). Never narrower than the @@ -256,9 +617,19 @@ pub fn execute_with_slots( variable, label, prop_eq, + prop_range, + text_pred, } => { - let keys = - index_scan_keys(memgraph, csr_segs, label.as_ref(), prop_eq, params, ctx); + let keys = index_scan_keys( + memgraph, + csr_segs, + label.as_ref(), + prop_eq, + prop_range, + text_pred, + params, + ctx, + ); let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); for row in &rows { for &key in &keys { @@ -278,6 +649,7 @@ pub fn execute_with_slots( direction, min_hops, max_hops, + optional, } => { let type_ids: Vec = edge_types .iter() @@ -315,8 +687,16 @@ pub fn execute_with_slots( for row in &rows { let src_key = match row.get(source) { Some(Value::Node(k)) => *k, - _ => continue, + _ => { + // W2-13: a Null/unbound source under OPTIONAL + // MATCH survives null-padded instead of dropping. + if *optional { + push_null_padded(row, target, edge_variable, &mut new_rows); + } + continue; + } }; + let row_start = new_rows.len(); if *max_hops <= 1 { // Single-hop expansion via SegmentMergeReader. @@ -393,6 +773,12 @@ pub fn execute_with_slots( } } } + + // W2-13: zero matches under OPTIONAL MATCH → the source + // row survives with target/edge bound to Null. + if *optional && new_rows.len() == row_start { + push_null_padded(row, target, edge_variable, &mut new_rows); + } } rows = new_rows; } @@ -414,7 +800,11 @@ pub fn execute_with_slots( }); } - PhysicalOp::Project { items, distinct } => { + PhysicalOp::Project { + items, + distinct, + rebind, + } => { columns = items .iter() .map(|item| { @@ -426,40 +816,74 @@ pub fn execute_with_slots( }) .collect(); - let mut projected: Vec> = rows - .iter() - .map(|row| { - items - .iter() - .map(|item| { - if matches!(item.expr, Expr::Star) { - let entries: Vec<(String, Value)> = row - .iter() - .map(|(k, v)| (k.to_owned(), v.clone())) - .collect(); - Value::Map(entries) - } else { - eval_expr( - &item.expr, - row, - memgraph, - params, - csr_segs, - ctx.snapshot_lsn, - ctx.decay, - ) - } - }) - .collect() - }) - .collect(); + // W2-12: aggregate items (count/sum/avg/min/max/collect) + // switch the projection into grouped-aggregation mode. + let aggregated = try_project_aggregate(items, rows.len(), |e, ri| { + eval_expr( + e, + &rows[ri], + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ) + }); + + let mut projected: Vec> = match aggregated { + Some(agg_rows) => agg_rows, + None => rows + .iter() + .map(|row| { + items + .iter() + .map(|item| { + if matches!(item.expr, Expr::Star) { + let entries: Vec<(String, Value)> = row + .iter() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); + Value::Map(entries) + } else { + eval_expr( + &item.expr, + row, + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ) + } + }) + .collect() + }) + .collect(), + }; if *distinct { dedup_rows(&mut projected); } - projected_rows = Some(projected); - rows.clear(); + if *rebind { + // W2-13 WITH: re-seed the variable-binding row stream + // with the projection outputs so later clauses (WHERE / + // ORDER BY / MATCH / RETURN) keep executing. `columns` + // stays set but the final RETURN overwrites it. + let mut new_rows = Vec::with_capacity(projected.len()); + for vals in projected { + let mut new_row = Row::seed(slot_table); + for (name, val) in columns.iter().zip(vals) { + new_row.insert(name, val); + } + new_rows.push(new_row); + } + rows = new_rows; + projected_rows = None; + } else { + projected_rows = Some(projected); + rows.clear(); + } } PhysicalOp::Sort { items } => { @@ -798,9 +1222,19 @@ pub fn execute_profile( variable, label, prop_eq, + prop_range, + text_pred, } => { - let keys = - index_scan_keys(memgraph, csr_segs, label.as_ref(), prop_eq, params, ctx); + let keys = index_scan_keys( + memgraph, + csr_segs, + label.as_ref(), + prop_eq, + prop_range, + text_pred, + params, + ctx, + ); let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); for row in &rows { for &key in &keys { @@ -820,6 +1254,7 @@ pub fn execute_profile( direction, min_hops, max_hops, + optional, } => { let type_ids: Vec = edge_types .iter() @@ -857,8 +1292,15 @@ pub fn execute_profile( for row in &rows { let src_key = match row.get(source) { Some(Value::Node(k)) => *k, - _ => continue, + _ => { + // W2-13 OPTIONAL MATCH (parity with main executor). + if *optional { + push_null_padded(row, target, edge_variable, &mut new_rows); + } + continue; + } }; + let row_start = new_rows.len(); if *max_hops <= 1 { reader.neighbors_into(src_key, &mut nb_seen, &mut nb_buf); @@ -931,6 +1373,10 @@ pub fn execute_profile( } } } + + if *optional && new_rows.len() == row_start { + push_null_padded(row, target, edge_variable, &mut new_rows); + } } rows = new_rows; } @@ -952,7 +1398,11 @@ pub fn execute_profile( }); } - PhysicalOp::Project { items, distinct } => { + PhysicalOp::Project { + items, + distinct, + rebind, + } => { columns = items .iter() .map(|item| { @@ -964,40 +1414,71 @@ pub fn execute_profile( }) .collect(); - let mut projected: Vec> = rows - .iter() - .map(|row| { - items - .iter() - .map(|item| { - if matches!(item.expr, Expr::Star) { - let entries: Vec<(String, Value)> = row - .iter() - .map(|(k, v)| (k.to_owned(), v.clone())) - .collect(); - Value::Map(entries) - } else { - eval_expr( - &item.expr, - row, - memgraph, - params, - csr_segs, - ctx.snapshot_lsn, - ctx.decay, - ) - } - }) - .collect() - }) - .collect(); + // W2-12: aggregate items switch into grouped-aggregation + // mode (shared with the non-profile executor). + let aggregated = try_project_aggregate(items, rows.len(), |e, ri| { + eval_expr( + e, + &rows[ri], + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ) + }); + + let mut projected: Vec> = match aggregated { + Some(agg_rows) => agg_rows, + None => rows + .iter() + .map(|row| { + items + .iter() + .map(|item| { + if matches!(item.expr, Expr::Star) { + let entries: Vec<(String, Value)> = row + .iter() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); + Value::Map(entries) + } else { + eval_expr( + &item.expr, + row, + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ) + } + }) + .collect() + }) + .collect(), + }; if *distinct { dedup_rows(&mut projected); } - projected_rows = Some(projected); - rows.clear(); + if *rebind { + // W2-13 WITH rebind (parity with main executor). + let mut new_rows = Vec::with_capacity(projected.len()); + for vals in projected { + let mut new_row = Row::seed(&slot_table); + for (name, val) in columns.iter().zip(vals) { + new_row.insert(name, val); + } + new_rows.push(new_row); + } + rows = new_rows; + projected_rows = None; + } else { + projected_rows = Some(projected); + rows.clear(); + } } PhysicalOp::Sort { items } => { diff --git a/src/graph/cypher/executor/write.rs b/src/graph/cypher/executor/write.rs index 9083230a4..6d6a46433 100644 --- a/src/graph/cypher/executor/write.rs +++ b/src/graph/cypher/executor/write.rs @@ -14,6 +14,14 @@ pub fn execute_mut( ) -> Result { let start = std::time::Instant::now(); + // Frozen CSR segments, cloned once (cheap Arc clones): the write path + // must SEE the frozen tier (scans/expands/filters) so SET/DELETE/MERGE + // can copy-up frozen rows instead of silently missing them. No freeze + // can run mid-query (freeze_and_compact is only driven by the ADDEDGE + // handler), so a snapshot at entry is safe. + let csr_segs: Vec> = + graph.segments.load().immutable.clone(); + let slot_table = SlotTable::from_plan(plan); let empty_table = SlotTable::default(); let empty_row = Row::seed(&empty_table); @@ -27,24 +35,25 @@ pub fn execute_mut( for op in &plan.operators { match op { - // The write path matches against the MUTABLE tier only (write - // operators mutate MutableNode in place; SET/DELETE on frozen - // rows is a separate copy-up design — known follow-up). The - // IndexScan arm therefore degrades to the same label scan; the - // residual Filter the planner emits keeps results exact. + // W2-2 copy-up: the write path scans BOTH tiers so SET/DELETE/ + // MERGE can target frozen rows (the mutation arms copy the row + // up into the write buffer first). The IndexScan arm degrades to + // the same label scan; the residual Filter the planner emits + // keeps results exact. PhysicalOp::NodeScan { variable, label } | PhysicalOp::IndexScan { variable, label, .. } => { let label_id = label.as_ref().map(|l| label_to_id(l.as_bytes())); - let mut new_rows = Vec::new(); + let committed = roaring::RoaringBitmap::new(); + let view = crate::graph::view::MergedNodeView::new(&graph.write_buf, &csr_segs); + let mut keys = Vec::new(); + view.for_each_visible_node(label_id, u64::MAX, 0, &committed, None, |k| { + keys.push(k) + }); + let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); for row in &rows { - for (key, node) in graph.write_buf.iter_nodes() { - if let Some(lid) = label_id { - if !node.labels.contains(&lid) { - continue; - } - } + for &key in &keys { let mut new_row = row.clone(); new_row.insert(variable, Value::Node(key)); new_rows.push(new_row); @@ -61,6 +70,7 @@ pub fn execute_mut( direction, min_hops, max_hops, + optional, } => { let type_ids: Vec = edge_types .iter() @@ -73,31 +83,52 @@ pub fn execute_mut( EdgeDirection::Both => Direction::Both, }; + // Merge reader: mutable adjacency + frozen CSR edges, so + // MATCH...SET/DELETE finds frozen endpoints. (Frozen edges + // carry a placeholder EdgeKey — SET/DELETE on a frozen EDGE + // stays a no-op; nodes are the copy-up unit.) + let edge_type_filter = if type_ids.len() == 1 { + Some(type_ids[0]) + } else { + None + }; + let reader = crate::graph::traversal::SegmentMergeReader::new( + Some(&graph.write_buf), + &csr_segs, + dir, + u64::MAX, + edge_type_filter, + ); + let mut nb_seen = crate::graph::fasthash::FxHashSet::default(); + let mut nb_buf: Vec = Vec::new(); + let mut new_rows = Vec::new(); for row in &rows { let src_key = match row.get(source) { Some(Value::Node(k)) => *k, - _ => continue, + _ => { + // W2-13 OPTIONAL MATCH (parity with the read path). + if *optional { + push_null_padded(row, target, edge_variable, &mut new_rows); + } + continue; + } }; + let row_start = new_rows.len(); if *max_hops <= 1 { - for (edge_key, neighbor_key) in - graph.write_buf.neighbors(src_key, dir, u64::MAX) - { - if !type_ids.is_empty() { - if let Some(edge) = graph.write_buf.get_edge(edge_key) { - if !type_ids.contains(&edge.edge_type) { - continue; - } - } + reader.neighbors_into(src_key, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { + if type_ids.len() > 1 && !type_ids.contains(&merged.edge_type) { + continue; } let mut new_row = row.clone(); - new_row.insert(target, Value::Node(neighbor_key)); + new_row.insert(target, Value::Node(merged.node)); // Phase 174 FIX-01: bind edge variable so DELETE r // can reference it. Previously ignored (`_`), which // made `DELETE r` a silent no-op. if let Some(evar) = edge_variable { - new_row.insert(evar, Value::Edge(edge_key)); + new_row.insert(evar, Value::Edge(merged.edge)); } new_rows.push(new_row); } @@ -115,18 +146,14 @@ pub fn execute_mut( for hop in 1..=capped_max_hops { let mut next_frontier = Vec::new(); for ¤t in &frontier { - for (edge_key, neighbor_key) in - graph.write_buf.neighbors(current, dir, u64::MAX) - { + reader.neighbors_into(current, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { + let neighbor_key = merged.node; if visited.contains(&neighbor_key) { continue; } - if !type_ids.is_empty() { - if let Some(edge) = graph.write_buf.get_edge(edge_key) { - if !type_ids.contains(&edge.edge_type) { - continue; - } - } + if type_ids.len() > 1 && !type_ids.contains(&merged.edge_type) { + continue; } visited.insert(neighbor_key); next_frontier.push(neighbor_key); @@ -150,6 +177,10 @@ pub fn execute_mut( } } } + + if *optional && new_rows.len() == row_start { + push_null_padded(row, target, edge_variable, &mut new_rows); + } } rows = new_rows; } @@ -157,13 +188,25 @@ pub fn execute_mut( PhysicalOp::Filter { expr } => { rows.retain(|row| { matches!( - eval_expr(expr, row, &graph.write_buf, params, &[], 0, None), + eval_expr( + expr, + row, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None + ), Value::Bool(true) ) }); } - PhysicalOp::Project { items, distinct } => { + PhysicalOp::Project { + items, + distinct, + rebind, + } => { columns = items .iter() .map(|item| { @@ -175,6 +218,8 @@ pub fn execute_mut( }) .collect(); + // W2-12 note: aggregate-mode projection (try_project_aggregate) + // is read-path only; the write path keeps per-row semantics. let mut projected: Vec> = rows .iter() .map(|row| { @@ -193,8 +238,8 @@ pub fn execute_mut( row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ) } @@ -207,8 +252,22 @@ pub fn execute_mut( dedup_rows(&mut projected); } - projected_rows = Some(projected); - rows.clear(); + if *rebind { + // W2-13 WITH rebind (parity with the read path). + let mut new_rows = Vec::with_capacity(projected.len()); + for vals in projected { + let mut new_row = Row::seed(&slot_table); + for (name, val) in columns.iter().zip(vals) { + new_row.insert(name, val); + } + new_rows.push(new_row); + } + rows = new_rows; + projected_rows = None; + } else { + projected_rows = Some(projected); + rows.clear(); + } } PhysicalOp::Sort { items } => { @@ -242,8 +301,24 @@ pub fn execute_mut( } else { rows.sort_by(|a, b| { for (expr, ascending) in items { - let va = eval_expr(expr, a, &graph.write_buf, params, &[], 0, None); - let vb = eval_expr(expr, b, &graph.write_buf, params, &[], 0, None); + let va = eval_expr( + expr, + a, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ); + let vb = eval_expr( + expr, + b, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ); let ord = compare_values(&va, &vb); let ord = if *ascending { ord } else { ord.reverse() }; if ord != std::cmp::Ordering::Equal { @@ -256,7 +331,15 @@ pub fn execute_mut( } PhysicalOp::Limit { count } => { - let n = match eval_expr(count, &empty_row, &graph.write_buf, params, &[], 0, None) { + let n = match eval_expr( + count, + &empty_row, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ) { Value::Int(n) if n >= 0 => n as usize, _ => 0, }; @@ -268,7 +351,15 @@ pub fn execute_mut( } PhysicalOp::Skip { count } => { - let n = match eval_expr(count, &empty_row, &graph.write_buf, params, &[], 0, None) { + let n = match eval_expr( + count, + &empty_row, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ) { Value::Int(n) if n >= 0 => n as usize, _ => 0, }; @@ -288,7 +379,15 @@ pub fn execute_mut( PhysicalOp::Unwind { expr, alias } => { let mut new_rows = Vec::new(); for row in &rows { - let val = eval_expr(expr, row, &graph.write_buf, params, &[], 0, None); + let val = eval_expr( + expr, + row, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ); if let Value::List(items) = val { for item in items { let mut new_row = row.clone(); @@ -322,8 +421,8 @@ pub fn execute_mut( &new_row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ); value_to_property_value(&val) @@ -395,39 +494,34 @@ pub fn execute_mut( row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ); if let Some(pv) = value_to_property_value(&val) { let pid = label_to_id(property.as_bytes()); - if let Some(node) = graph.write_buf.get_node_mut(*nk) { - // Phase 174 FIX-01: snapshot old value BEFORE - // mutating so TXN.ABORT can restore it. - let old_value = node - .properties - .iter() - .find(|(k, _)| *k == pid) - .map(|(_, v)| v.clone()); + // W2-2: frozen target → copy the row up + // into the write buffer, then mutate. + graph.copy_up_node(*nk); + if graph.write_buf.get_node(*nk).is_some() { + // Phase 174 FIX-01: snapshot new value + // BEFORE the move so TXN.ABORT can + // restore it. `set_node_property` is + // the single source of truth for + // node-property mutation — it keeps + // the mutable-tier property index + // (Task #31) in sync and returns the + // old value the undo record needs. + let new_value = pv.clone(); + let old_value = + graph.write_buf.set_node_property(*nk, pid, pv); mutations.push(MutationRecord::SetProperty { entity_id: nk.data().as_ffi(), is_node: true, key: pid, old_value, + new_value, }); - - // Update existing or append. - let mut found = false; - for entry in node.properties.iter_mut() { - if entry.0 == pid { - entry.1 = pv.clone(); - found = true; - break; - } - } - if !found { - node.properties.push((pid, pv)); - } properties_set += 1; } } @@ -436,9 +530,17 @@ pub fn execute_mut( SetItem::Label { variable, label } => { if let Some(Value::Node(nk)) = row.get(variable) { let lid = label_to_id(label.as_bytes()); + graph.copy_up_node(*nk); if let Some(node) = graph.write_buf.get_node_mut(*nk) { if !node.labels.contains(&lid) { node.labels.push(lid); + // W2-9: record for WAL durability + // (idempotent — only when newly + // added). + mutations.push(MutationRecord::SetLabel { + node_id: nk.data().as_ffi(), + label: lid, + }); } } } @@ -452,9 +554,21 @@ pub fn execute_mut( let _ = detach; // Detach is always implied for MemGraph soft-delete. for row in &rows { for expr in exprs { - let val = eval_expr(expr, row, &graph.write_buf, params, &[], 0, None); + let val = eval_expr( + expr, + row, + &graph.write_buf, + params, + &csr_segs, + u64::MAX, + None, + ); match val { Value::Node(nk) => { + // W2-2: frozen target → copy the row up so the + // soft-delete lands in the write buffer as a + // TOMBSTONE shadowing the frozen row. + graph.copy_up_node(nk); // Phase 174 FIX-01: snapshot node state BEFORE // soft-delete so TXN.ABORT can un-delete. if let Some(node) = graph.write_buf.get_node(nk) { @@ -522,8 +636,8 @@ pub fn execute_mut( &new_row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ); value_to_property_value(&val) @@ -531,30 +645,10 @@ pub fn execute_mut( }) .collect(); - // Search for existing node matching labels + properties. - let found = graph - .write_buf - .iter_nodes() - .find(|(_, node)| { - // All required labels must be present. - for &lid in &label_ids { - if !node.labels.contains(&lid) { - return false; - } - } - // All required properties must match. - for (pid, pval) in &match_props { - let has_match = node - .properties - .iter() - .any(|(np, nv)| *np == *pid && *nv == *pval); - if !has_match { - return false; - } - } - true - }) - .map(|(k, _)| k); + // Search BOTH tiers for an existing node matching + // labels + properties (a mutable-only search would + // duplicate frozen nodes on every MERGE). + let found = find_node_merged(graph, &csr_segs, &label_ids, &match_props); if let Some(existing_key) = found { // MATCH path: bind variable and apply on_match. @@ -564,7 +658,8 @@ pub fn execute_mut( apply_set_items( on_match, &new_row, - &mut graph.write_buf, + graph, + &csr_segs, params, &mut properties_set, Some(&mut mutations), @@ -588,10 +683,14 @@ pub fn execute_mut( apply_set_items( on_create, &new_row, - &mut graph.write_buf, + graph, + &csr_segs, params, &mut properties_set, - None, + // W2-9: ON CREATE SET needs mutation records + // too — the CreateNode WAL snapshot predates + // the SET (see apply_set_items doc). + Some(&mut mutations), ); } } else if !pattern.edges.is_empty() && pattern.nodes.len() >= 2 { @@ -600,11 +699,11 @@ pub fn execute_mut( let dst_pn = &pattern.nodes[1]; let pe = &pattern.edges[0]; - // Resolve or find source node. + // Resolve or find source node (both tiers). let src_key = - resolve_or_find_node(src_pn, &new_row, &graph.write_buf, params); + resolve_or_find_node(src_pn, &new_row, graph, &csr_segs, params); let dst_key = - resolve_or_find_node(dst_pn, &new_row, &graph.write_buf, params); + resolve_or_find_node(dst_pn, &new_row, graph, &csr_segs, params); let edge_type_id = pe .edge_types @@ -614,17 +713,18 @@ pub fn execute_mut( match (src_key, dst_key) { (Some(sk), Some(dk)) => { - // Check if edge exists. - let edge_exists = graph - .write_buf - .neighbors(sk, Direction::Outgoing, u64::MAX) - .any(|(ek, nk)| { - nk == dk - && graph - .write_buf - .get_edge(ek) - .map_or(false, |e| e.edge_type == edge_type_id) - }); + // Check if edge exists in EITHER tier (frozen + // edges live in CSR adjacency). + let edge_exists = { + let reader = crate::graph::traversal::SegmentMergeReader::new( + Some(&graph.write_buf), + &csr_segs, + Direction::Outgoing, + u64::MAX, + Some(edge_type_id), + ); + reader.neighbors(sk).iter().any(|m| m.node == dk) + }; if edge_exists { // Bind variables. @@ -637,14 +737,15 @@ pub fn execute_mut( apply_set_items( on_match, &new_row, - &mut graph.write_buf, + graph, + &csr_segs, params, &mut properties_set, Some(&mut mutations), ); } else { // Create edge. - if let Ok(ek) = graph.write_buf.add_edge( + if let Ok(ek) = graph.write_buf.add_edge_across_tiers( sk, dk, edge_type_id, @@ -670,10 +771,12 @@ pub fn execute_mut( apply_set_items( on_create, &new_row, - &mut graph.write_buf, + graph, + &csr_segs, params, &mut properties_set, - None, + // W2-9: see apply_set_items doc. + Some(&mut mutations), ); } } @@ -696,8 +799,8 @@ pub fn execute_mut( &new_row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ); value_to_property_value(&val) @@ -733,8 +836,8 @@ pub fn execute_mut( &new_row, &graph.write_buf, params, - &[], - 0, + &csr_segs, + u64::MAX, None, ); value_to_property_value(&val) @@ -753,11 +856,14 @@ pub fn execute_mut( }); nk }; - if let Ok(ek) = - graph - .write_buf - .add_edge(sk, dk, edge_type_id, 1.0, None, lsn) - { + if let Ok(ek) = graph.write_buf.add_edge_across_tiers( + sk, + dk, + edge_type_id, + 1.0, + None, + lsn, + ) { mutations.push(MutationRecord::CreateEdge { edge_id: ek.data().as_ffi(), src_id: sk.data().as_ffi(), @@ -776,7 +882,8 @@ pub fn execute_mut( apply_set_items( on_create, &new_row, - &mut graph.write_buf, + graph, + &csr_segs, params, &mut properties_set, None, @@ -852,12 +959,20 @@ pub fn execute_mut( /// /// Phase 174 FIX-01: accepts an optional `mutations` vec to emit /// `MutationRecord::SetProperty` records for MERGE ON MATCH SET rollback. -/// Pass `None` for ON CREATE SET paths (no rollback needed for freshly -/// created nodes — they are removed entirely by the CreateNode intent). +/// W2-9: mutation records now also drive WAL generation, so ON CREATE SET +/// paths must pass `Some` too — the CreateNode WAL record snapshots +/// properties BEFORE the SET applies, and without a SetProperty record the +/// ON CREATE SET values are silently lost on kill -9. (Rollback stays +/// correct: the extra RestoreProperty undo is a no-op on a node the +/// CreateNode intent removes entirely.) +/// +/// W2-2: takes the whole `NamedGraph` (not just the write buffer) so a +/// frozen target node can be copied up before the in-place mutation. pub(crate) fn apply_set_items( items: &[SetItem], row: &Row<'_>, - memgraph: &mut crate::graph::memgraph::MemGraph, + graph: &mut NamedGraph, + csr_segs: &[std::sync::Arc], params: &HashMap, properties_set: &mut u64, mut mutations: Option<&mut Vec>, @@ -870,36 +985,36 @@ pub(crate) fn apply_set_items( value, } => { if let Some(Value::Node(nk)) = row.get(variable) { - let val = eval_expr(value, row, memgraph, params, &[], 0, None); + let val = eval_expr( + value, + row, + &graph.write_buf, + params, + csr_segs, + u64::MAX, + None, + ); if let Some(pv) = value_to_property_value(&val) { let pid = label_to_id(property.as_bytes()); - if let Some(node) = memgraph.get_node_mut(*nk) { - // Phase 174 FIX-01: snapshot old value for rollback. - if let Some(muts) = mutations.as_mut() { - let old_value = node - .properties - .iter() - .find(|(k, _)| *k == pid) - .map(|(_, v)| v.clone()); + graph.copy_up_node(*nk); + if graph.write_buf.get_node(*nk).is_some() { + // Phase 174 FIX-01: snapshot new value for rollback + // BEFORE the move, only when a caller wants undo + // tracking. `set_node_property` is the single + // source of truth for node-property mutation — it + // keeps the mutable-tier property index (Task #31) + // in sync regardless of `mutations` tracking. + let new_value = mutations.is_some().then(|| pv.clone()); + let old_value = graph.write_buf.set_node_property(*nk, pid, pv); + if let (Some(muts), Some(new_value)) = (mutations.as_mut(), new_value) { muts.push(MutationRecord::SetProperty { entity_id: nk.data().as_ffi(), is_node: true, key: pid, old_value, + new_value, }); } - - let mut found = false; - for entry in node.properties.iter_mut() { - if entry.0 == pid { - entry.1 = pv.clone(); - found = true; - break; - } - } - if !found { - node.properties.push((pid, pv)); - } *properties_set += 1; } } @@ -908,9 +1023,18 @@ pub(crate) fn apply_set_items( SetItem::Label { variable, label } => { if let Some(Value::Node(nk)) = row.get(variable) { let lid = label_to_id(label.as_bytes()); - if let Some(node) = memgraph.get_node_mut(*nk) { + graph.copy_up_node(*nk); + if let Some(node) = graph.write_buf.get_node_mut(*nk) { if !node.labels.contains(&lid) { node.labels.push(lid); + // W2-9: WAL durability (idempotent — only when + // newly added). + if let Some(muts) = mutations.as_mut() { + muts.push(MutationRecord::SetLabel { + node_id: nk.data().as_ffi(), + label: lid, + }); + } } } } @@ -919,12 +1043,74 @@ pub(crate) fn apply_set_items( } } +/// Find a node matching all `label_ids` + `match_props` across BOTH tiers. +/// Mutable tier first (direct field access, no clones), then frozen segments +/// via `MergedNodeView` (which skips rows shadowed by copy-up entries). +pub(crate) fn find_node_merged( + graph: &NamedGraph, + csr_segs: &[std::sync::Arc], + label_ids: &[u16], + match_props: &[(u16, PropertyValue)], +) -> Option { + let matches_mutable = |node: &crate::graph::types::MutableNode| { + label_ids.iter().all(|lid| node.labels.contains(lid)) + && match_props.iter().all(|(pid, pval)| { + node.properties + .iter() + .any(|(np, nv)| *np == *pid && *nv == *pval) + }) + }; + if let Some(k) = graph + .write_buf + .iter_nodes() + .find(|(_, node)| matches_mutable(node)) + .map(|(k, _)| k) + { + return Some(k); + } + + let view = crate::graph::view::MergedNodeView::new(&graph.write_buf, csr_segs); + let committed = roaring::RoaringBitmap::new(); + let mut found = None; + view.for_each_visible_node( + label_ids.first().copied(), + u64::MAX, + 0, + &committed, + None, + |k| { + if found.is_some() { + return; + } + // Mutable tier already searched above. + if graph.write_buf.get_node(k).is_some() { + return; + } + let Some(labels) = view.labels(k) else { + return; + }; + if !label_ids.iter().all(|lid| labels.contains(lid)) { + return; + } + for (pid, pval) in match_props { + match view.property(k, *pid) { + Some(v) if v == *pval => {} + _ => return, + } + } + found = Some(k); + }, + ); + found +} + /// Resolve a pattern node: if it's a bound variable in the row, return that key. -/// Otherwise, search the memgraph for a matching node by labels + properties. +/// Otherwise, search BOTH tiers for a matching node by labels + properties. pub(crate) fn resolve_or_find_node( pn: &PatternNode, row: &Row<'_>, - memgraph: &crate::graph::memgraph::MemGraph, + graph: &NamedGraph, + csr_segs: &[std::sync::Arc], params: &HashMap, ) -> Option { // Check if already bound. @@ -944,29 +1130,18 @@ pub(crate) fn resolve_or_find_node( .properties .iter() .filter_map(|(name, expr)| { - let val = eval_expr(expr, row, memgraph, params, &[], 0, None); + let val = eval_expr( + expr, + row, + &graph.write_buf, + params, + csr_segs, + u64::MAX, + None, + ); value_to_property_value(&val).map(|pv| (label_to_id(name.as_bytes()), pv)) }) .collect(); - memgraph - .iter_nodes() - .find(|(_, node)| { - for &lid in &label_ids { - if !node.labels.contains(&lid) { - return false; - } - } - for (pid, pval) in &match_props { - let has_match = node - .properties - .iter() - .any(|(np, nv)| *np == *pid && *nv == *pval); - if !has_match { - return false; - } - } - true - }) - .map(|(k, _)| k) + find_node_merged(graph, csr_segs, &label_ids, &match_props) } diff --git a/src/graph/cypher/lexer.rs b/src/graph/cypher/lexer.rs index e796dd5a5..18bcd3836 100644 --- a/src/graph/cypher/lexer.rs +++ b/src/graph/cypher/lexer.rs @@ -74,6 +74,15 @@ pub enum Token<'a> { Desc, #[token(b"ON", ignore(case))] On, + // --- P3 text predicates (design part B): CONTAINS is a single keyword; + // STARTS WITH / ENDS WITH are two keywords parsed together (WITH is + // reused from the WITH clause -- no new token needed for it). + #[token(b"CONTAINS", ignore(case))] + Contains, + #[token(b"STARTS", ignore(case))] + Starts, + #[token(b"ENDS", ignore(case))] + Ends, // --- Identifiers --- // Must come after keywords so logos prefers keyword matches. @@ -376,6 +385,19 @@ mod tests { assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::ArrowRight)); } + #[test] + fn test_text_predicate_keywords() { + // P3 design part B: CONTAINS / STARTS WITH / ENDS WITH keywords, + // case-insensitive like every other Cypher keyword. + let input = b"CONTAINS starts with Ends With"; + let mut lexer = Lexer::new(input); + assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::Contains)); + assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::Starts)); + assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::With)); + assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::Ends)); + assert_eq!(lexer.next_token().map(|t| t.token), Some(Token::With)); + } + #[test] fn test_comments_skipped() { let input = b"MATCH // line comment\nRETURN /* block */"; diff --git a/src/graph/cypher/mod.rs b/src/graph/cypher/mod.rs index 751993ff8..beacff40b 100644 --- a/src/graph/cypher/mod.rs +++ b/src/graph/cypher/mod.rs @@ -14,11 +14,13 @@ pub mod lexer; pub mod parameterize; pub mod parser; pub mod planner; +pub mod result_cache; pub use ast::{Clause, CypherQuery, Expr}; pub use executor::{ExecResult, OpProfile, ProfileResult, Value}; pub use parser::{CypherError, DEFAULT_MAX_NESTING_DEPTH, Parser}; pub use planner::{CostEstimate, PhysicalPlan, PlanCache, Strategy}; +pub use result_cache::{ResultCache, ResultCacheKey}; /// Parse a Cypher query from a byte slice. /// diff --git a/src/graph/cypher/parameterize.rs b/src/graph/cypher/parameterize.rs index 01bbbc1d5..d8d17b02b 100644 --- a/src/graph/cypher/parameterize.rs +++ b/src/graph/cypher/parameterize.rs @@ -137,7 +137,7 @@ fn literal_value(tok: &Token) -> Option { let inner = &s[1..s.len() - 1]; core::str::from_utf8(inner) .ok() - .map(|t| Value::String(t.to_owned())) + .map(|t| Value::String(bytes::Bytes::copy_from_slice(t.as_bytes()))) } _ => None, } diff --git a/src/graph/cypher/parser/expr.rs b/src/graph/cypher/parser/expr.rs index 152e9dc56..48ddb6ee1 100644 --- a/src/graph/cypher/parser/expr.rs +++ b/src/graph/cypher/parser/expr.rs @@ -96,12 +96,26 @@ impl<'a> Parser<'a> { Some(BinaryOperator::GreaterEqual) } else if self.peek_is(|t| matches!(t, Token::RegexMatch)) { Some(BinaryOperator::RegexMatch) + } else if self.peek_is(|t| matches!(t, Token::Contains)) { + Some(BinaryOperator::Contains) + } else if self.peek_is(|t| matches!(t, Token::Starts)) + && self.peek2_is(|t| matches!(t, Token::With)) + { + Some(BinaryOperator::StartsWith) + } else if self.peek_is(|t| matches!(t, Token::Ends)) + && self.peek2_is(|t| matches!(t, Token::With)) + { + Some(BinaryOperator::EndsWith) } else { None }; if let Some(op) = op { self.advance(); + // STARTS WITH / ENDS WITH are two keywords -- consume WITH too. + if matches!(op, BinaryOperator::StartsWith | BinaryOperator::EndsWith) { + self.advance(); + } let right = self.parse_addition()?; left = Expr::BinaryOp { left: Box::new(left), diff --git a/src/graph/cypher/parser/mod.rs b/src/graph/cypher/parser/mod.rs index d24a65ffa..e9053ce02 100644 --- a/src/graph/cypher/parser/mod.rs +++ b/src/graph/cypher/parser/mod.rs @@ -479,6 +479,15 @@ impl<'a> Parser<'a> { self.lexer.peek().map_or(false, |t| pred(&t.token)) } + /// Like `peek_is`, but for the token AFTER the next one -- needed for + /// two-keyword operators (`STARTS WITH`, `ENDS WITH`, P3 design part B). + pub(super) fn peek2_is(&mut self, pred: F) -> bool + where + F: FnOnce(&Token<'_>) -> bool, + { + self.lexer.peek2().map_or(false, |t| pred(&t.token)) + } + pub(super) fn peek_token_ref(&mut self) -> Result, CypherError> { self.lexer .peek() @@ -731,6 +740,74 @@ mod tests { } } + // ----------------------------------------------------------------------- + // Text predicates (P3 design part B): CONTAINS / STARTS WITH / ENDS WITH + // ----------------------------------------------------------------------- + + #[test] + fn test_parse_contains_operator() { + let q = parse("MATCH (n) WHERE n.bio CONTAINS 'rust' RETURN n").expect("parse failed"); + if let Clause::Where(w) = &q.clauses[1] { + assert!(matches!( + w.expr, + Expr::BinaryOp { + op: BinaryOperator::Contains, + .. + } + )); + } else { + panic!("expected Where clause"); + } + } + + #[test] + fn test_parse_starts_with() { + let q = parse("MATCH (n) WHERE n.name STARTS WITH 'Al' RETURN n").expect("parse failed"); + if let Clause::Where(w) = &q.clauses[1] { + assert!(matches!( + w.expr, + Expr::BinaryOp { + op: BinaryOperator::StartsWith, + .. + } + )); + } else { + panic!("expected Where clause"); + } + } + + #[test] + fn test_parse_ends_with() { + let q = parse("MATCH (n) WHERE n.name ENDS WITH 'ce' RETURN n").expect("parse failed"); + if let Clause::Where(w) = &q.clauses[1] { + assert!(matches!( + w.expr, + Expr::BinaryOp { + op: BinaryOperator::EndsWith, + .. + } + )); + } else { + panic!("expected Where clause"); + } + } + + #[test] + fn test_parse_text_predicate_case_insensitive() { + let q = parse("MATCH (n) WHERE n.name starts with 'Al' RETURN n").expect("parse failed"); + if let Clause::Where(w) = &q.clauses[1] { + assert!(matches!( + w.expr, + Expr::BinaryOp { + op: BinaryOperator::StartsWith, + .. + } + )); + } else { + panic!("expected Where clause"); + } + } + // ----------------------------------------------------------------------- // Parameters (injection prevention) // ----------------------------------------------------------------------- diff --git a/src/graph/cypher/planner.rs b/src/graph/cypher/planner.rs index b6a4b1921..d3453ba33 100644 --- a/src/graph/cypher/planner.rs +++ b/src/graph/cypher/planner.rs @@ -7,6 +7,7 @@ //! The cost estimator selects between graph-first and vector-first strategies //! based on per-graph `GraphStats` (degree distribution, node/edge counts). +use std::collections::HashSet; use std::sync::Arc; use crate::graph::cypher::ast::*; @@ -20,6 +21,31 @@ pub struct PhysicalPlan { pub operators: Vec, } +/// Ordering comparison for an `IndexScan` range conjunct (W2-3). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RangeCmp { + /// `prop > threshold` + Gt, + /// `prop >= threshold` + Gte, + /// `prop < threshold` + Lt, + /// `prop <= threshold` + Lte, +} + +impl RangeCmp { + /// Flip for the mirrored form (`threshold < prop` ⇔ `prop > threshold`). + fn flipped(self) -> Self { + match self { + RangeCmp::Gt => RangeCmp::Lt, + RangeCmp::Gte => RangeCmp::Lte, + RangeCmp::Lt => RangeCmp::Gt, + RangeCmp::Lte => RangeCmp::Gte, + } + } +} + /// Individual physical operators in the execution pipeline. #[derive(Debug, Clone)] pub enum PhysicalOp { @@ -43,6 +69,22 @@ pub enum PhysicalOp { /// (property name, value expression) equality conjuncts. Expressions /// are literals or parameters, resolved by the executor per run. prop_eq: Vec<(String, Expr)>, + /// (property name, comparison, threshold expression) range conjuncts + /// extracted from top-level `WHERE` AND-chains (`n.p > 5`, + /// `10 > n.p`, `n.p >= $t`). Numeric-index pruning hints with the + /// same SUPERSET contract as `prop_eq` — the residual Filter + /// downstream stays authoritative. W2-3. + prop_range: Vec<(String, RangeCmp, Expr)>, + /// (property name, text operator, pattern expression) text-predicate + /// conjuncts extracted from top-level `WHERE` AND-chains + /// (`n.p CONTAINS 'x'`, `n.p STARTS WITH $prefix`, `n.p =~ '.*x.*'`). + /// P3 design part B. Pruned via `SegmentTextIndex::candidate_rows` — + /// a PRESENCE-only superset (see `text_index.rs` module docs for why + /// token-level pruning is unsound for substring/prefix/suffix + /// predicates), same SUPERSET contract as `prop_eq`/`prop_range`; + /// the pattern expression itself is carried for documentation/future + /// use but is NOT evaluated for pruning purposes. + text_pred: Vec<(String, BinaryOperator, Expr)>, }, /// Expand along edges from a source variable. /// @@ -60,6 +102,11 @@ pub enum PhysicalOp { direction: EdgeDirection, min_hops: u32, max_hops: u32, + /// W2-13 OPTIONAL MATCH: a source row whose expansion yields zero + /// matches survives with `target` (and `edge_variable`) bound to + /// Null instead of being dropped. A Null/unbound source also + /// null-pads instead of being filtered out. + optional: bool, }, /// Filter rows by a predicate expression. Filter { expr: Expr }, @@ -67,6 +114,11 @@ pub enum PhysicalOp { Project { items: Vec, distinct: bool, + /// W2-13 WITH: instead of terminating the pipeline into positional + /// output rows, re-seed the variable-binding row stream with the + /// projection's outputs (alias or expression text) so later + /// MATCH/WHERE/RETURN clauses keep running. RETURN keeps `false`. + rebind: bool, }, /// Sort by expressions. Sort { items: Vec<(Expr, bool)> }, @@ -141,12 +193,16 @@ pub struct CachedPlan { /// literal values and are always empty here -- callers must re-derive /// params for that specific query text via `parameterize()`. pub auto_params: Arc>, + /// Whether the cached plan is read-only (W2-7 caches WRITE plans too) — + /// callers MUST route on this flag (see `PlanCache` docs). + pub read_only: bool, } struct CacheEntry { plan: Arc, slots: Arc, auto_params: Arc>, + read_only: bool, used: u64, } @@ -168,9 +224,12 @@ struct CacheEntry { /// deliberately hash-flooded cache degrades to O(max_entries) per lookup, /// not unbounded blowup. /// -/// Invariant: only READ-ONLY plans may be inserted. A cache hit in -/// `graph_query_or_write` routes straight to the read path without -/// re-parsing, so a cached write plan would mis-route. +/// Entries carry a `read_only` flag (W2-7 caches WRITE plans too). The +/// safety invariant moved from "only read-only plans may be inserted" to +/// "a read-path hit must check the flag": `graph_query_or_write` routes a +/// read-only hit to the read path and a write hit to the write path, both +/// with zero parse/compile work; the pure read handlers treat a write hit +/// as a miss. pub struct PlanCache { cache: FxHashMap, max_entries: usize, @@ -198,6 +257,7 @@ impl PlanCache { plan: e.plan.clone(), slots: e.slots.clone(), auto_params: e.auto_params.clone(), + read_only: e.read_only, } }) } @@ -205,12 +265,18 @@ impl PlanCache { /// Insert a plan under the literal-normalized hash, evicting the /// least-recently-used entry at capacity. No `auto_params` are cached /// (see `PlanCache` docs) -- callers re-derive them per query text. + /// `read_only` marks whether the plan may run on the read path (W2-7). /// /// Returns the freshly-built `SlotTable` so the caller can execute /// immediately without rebuilding it a second time. - pub fn insert(&mut self, hash: u64, plan: Arc) -> Arc { + pub fn insert( + &mut self, + hash: u64, + plan: Arc, + read_only: bool, + ) -> Arc { let slots = Arc::new(SlotTable::from_plan(&plan)); - self.put(hash, plan, slots.clone(), Arc::new(Vec::new())); + self.put(hash, plan, slots.clone(), Arc::new(Vec::new()), read_only); slots } @@ -227,6 +293,7 @@ impl PlanCache { normalized_hash: u64, plan: Arc, raw_auto_params: Vec<(String, Value)>, + read_only: bool, ) -> Arc { let slots = Arc::new(SlotTable::from_plan(&plan)); self.put( @@ -234,9 +301,16 @@ impl PlanCache { plan.clone(), slots.clone(), Arc::new(raw_auto_params), + read_only, ); if normalized_hash != raw_hash { - self.put(normalized_hash, plan, slots.clone(), Arc::new(Vec::new())); + self.put( + normalized_hash, + plan, + slots.clone(), + Arc::new(Vec::new()), + read_only, + ); } slots } @@ -247,6 +321,7 @@ impl PlanCache { plan: Arc, slots: Arc, auto_params: Arc>, + read_only: bool, ) { self.tick += 1; if self.cache.len() >= self.max_entries && !self.cache.contains_key(&hash) { @@ -267,6 +342,7 @@ impl PlanCache { plan, slots, auto_params, + read_only, used: self.tick, }, ); @@ -434,16 +510,39 @@ pub fn select_strategy( /// Strategy selection is done separately via `select_strategy()`. pub fn compile(query: &CypherQuery) -> Result { let mut ops = Vec::new(); + // W2-13: variables bound so far, for OPTIONAL MATCH shape validation. + // WITH resets this set to its own outputs (Cypher scoping). + let mut bound: HashSet = HashSet::new(); + // W2-13: once a WITH has run, range-conjunct extraction must stop — a + // post-WITH WHERE is a HAVING over (possibly aggregated) projections + // and must never prune pre-aggregation scans. + let mut saw_with = false; for clause in &query.clauses { match clause { Clause::Match(m) => { - compile_match(m, &mut ops)?; + compile_match(m, &mut ops, &mut bound)?; } Clause::ShortestPathMatch(sp) => { compile_shortest_path_match(sp, &mut ops); + for node in [&sp.src, &sp.dst] { + if let Some(v) = &node.variable { + bound.insert(v.clone()); + } + } + bound.insert(sp.path_var.clone()); } Clause::Where(w) => { + // W2-3: top-level AND conjuncts of the form + // `var.prop literal|param` upgrade var's scan to a + // range IndexScan BEFORE the residual Filter is pushed + // (the full WHERE stays — the index only prunes). + if !saw_with { + extract_range_conjuncts(&w.expr, &mut ops); + // P3 design part B: same treatment for text-predicate + // conjuncts (`CONTAINS`/`STARTS WITH`/`ENDS WITH`/`=~`). + extract_text_conjuncts(&w.expr, &mut ops); + } ops.push(PhysicalOp::Filter { expr: w.expr.clone(), }); @@ -452,12 +551,16 @@ pub fn compile(query: &CypherQuery) -> Result { ops.push(PhysicalOp::Project { items: r.items.clone(), distinct: r.distinct, + rebind: false, }); } Clause::Create(c) => { ops.push(PhysicalOp::CreatePattern { patterns: c.patterns.clone(), }); + for p in &c.patterns { + bind_pattern_vars(p, &mut bound); + } } Clause::Delete(d) => { ops.push(PhysicalOp::DeleteEntities { @@ -476,18 +579,36 @@ pub fn compile(query: &CypherQuery) -> Result { on_create: m.on_create.clone(), on_match: m.on_match.clone(), }); + bind_pattern_vars(&m.pattern, &mut bound); } Clause::With(w) => { + // W2-13: WITH is a rebinding projection — later clauses keep + // executing on the projected variable stream. + if w.items.iter().any(|it| matches!(it.expr, Expr::Star)) { + return Err(PlanError::Unsupported( + "WITH * is not yet supported — list the variables explicitly".to_string(), + )); + } ops.push(PhysicalOp::Project { items: w.items.clone(), distinct: w.distinct, + rebind: true, }); + // Cypher scoping: only the WITH outputs remain in scope. + bound.clear(); + for item in &w.items { + if let Some(name) = with_output_name(item) { + bound.insert(name); + } + } + saw_with = true; } Clause::Unwind(u) => { ops.push(PhysicalOp::Unwind { expr: u.expr.clone(), alias: u.alias.clone(), }); + bound.insert(u.alias.clone()); } Clause::Call(c) => { ops.push(PhysicalOp::ProcedureCall { @@ -517,6 +638,35 @@ pub fn compile(query: &CypherQuery) -> Result { Ok(PhysicalPlan { operators: ops }) } +/// The scope name a WITH item introduces, for OPTIONAL MATCH validation. +/// +/// Alias wins; a bare identifier passes through under its own name. Any +/// other unaliased expression produces an output column that no later +/// identifier can reference, so it contributes nothing to the bound set. +fn with_output_name(item: &ReturnItem) -> Option { + if let Some(alias) = &item.alias { + return Some(alias.clone()); + } + if let Expr::Ident(name) = &item.expr { + return Some(name.clone()); + } + None +} + +/// Record every variable a pattern binds (nodes and edges). +fn bind_pattern_vars(pattern: &Pattern, bound: &mut HashSet) { + for node in &pattern.nodes { + if let Some(v) = &node.variable { + bound.insert(v.clone()); + } + } + for edge in &pattern.edges { + if let Some(v) = &edge.variable { + bound.insert(v.clone()); + } + } +} + /// Compile a MATCH clause into scan + expand operators. /// /// Returns `Err(PlanError::Unsupported)` if a variable-length edge pattern @@ -525,11 +675,27 @@ pub fn compile(query: &CypherQuery) -> Result { /// predicates silently evaluate against Null — producing wrong results. /// This gate is temporary: Phase 179 (MVCC-02) will implement `Value::Path` /// binding and remove this restriction. -fn compile_match(m: &MatchClause, ops: &mut Vec) -> Result<(), PlanError> { +/// +/// W2-13 OPTIONAL MATCH compiles to an `Expand { optional: true }` and is +/// restricted to its dominant shape — a single relationship expanding from +/// a previously bound bare variable, no inline properties on the optional +/// target. Everything else (standalone patterns, multi-relationship chains +/// whose whole-pattern null semantics need grouped execution, inline +/// property predicates that must not drop null-padded rows) is rejected +/// loudly instead of silently behaving like an inner MATCH. +fn compile_match( + m: &MatchClause, + ops: &mut Vec, + bound: &mut HashSet, +) -> Result<(), PlanError> { + if m.optional { + return compile_optional_match(m, ops, bound); + } for pattern in &m.patterns { if pattern.nodes.is_empty() { continue; } + bind_pattern_vars(pattern, bound); // First node becomes a scan (index-backed when inline equality // properties can drive a lookup). @@ -575,6 +741,7 @@ fn compile_match(m: &MatchClause, ops: &mut Vec) -> Result<(), PlanE direction: edge.direction, min_hops, max_hops, + optional: false, }); // Inline properties on the expanded target node — filter right after // the Expand that BINDS it (after, never before: the var is unbound @@ -589,6 +756,97 @@ fn compile_match(m: &MatchClause, ops: &mut Vec) -> Result<(), PlanE Ok(()) } +/// Compile `OPTIONAL MATCH` (W2-13). See [`compile_match`] for the supported +/// shape and the rationale for each rejection. +fn compile_optional_match( + m: &MatchClause, + ops: &mut Vec, + bound: &mut HashSet, +) -> Result<(), PlanError> { + for pattern in &m.patterns { + let Some(first) = pattern.nodes.first() else { + continue; + }; + let Some(first_var) = &first.variable else { + return Err(PlanError::Unsupported( + "OPTIONAL MATCH must expand from a previously bound variable — \ + name the first pattern node" + .to_string(), + )); + }; + if !bound.contains(first_var) { + return Err(PlanError::Unsupported(format!( + "OPTIONAL MATCH must expand from a previously bound variable; \ + '{first_var}' is not bound. Standalone OPTIONAL MATCH patterns \ + are not yet supported." + ))); + } + if !first.labels.is_empty() || !first.properties.is_empty() { + return Err(PlanError::Unsupported(format!( + "OPTIONAL MATCH: labels/properties on the already-bound variable \ + '{first_var}' are not yet supported — constrain it in the \ + binding MATCH instead." + ))); + } + // `OPTIONAL MATCH (a)` with `a` bound re-matches an existing row: + // a no-op. + if pattern.edges.is_empty() { + continue; + } + if pattern.edges.len() > 1 { + return Err(PlanError::Unsupported( + "OPTIONAL MATCH with more than one relationship is not yet \ + supported (whole-pattern null semantics need grouped \ + execution) — split into single-hop OPTIONAL MATCH clauses" + .to_string(), + )); + } + let edge = &pattern.edges[0]; + // CYP-06 gate applies here too (see compile_match). + if edge.var_length.is_some() && edge.variable.is_some() { + let var_name = edge.variable.as_deref().unwrap_or("?"); + return Err(PlanError::Unsupported(format!( + "CYP-06: Multi-hop edge variable binding -[{var_name}*m..n]- is not yet \ + supported. Remove the edge variable '{var_name}' or use a single-hop \ + pattern. Tracked: MVCC-02 (Phase 179)." + ))); + } + // Parser invariant: nodes.len() == edges.len() + 1; stay panic-free + // on a malformed AST anyway. + let Some(target) = pattern.nodes.get(1) else { + continue; + }; + if !target.properties.is_empty() { + return Err(PlanError::Unsupported( + "OPTIONAL MATCH: inline properties on the optional target are \ + not yet supported (a post-filter would drop null-padded rows) \ + — use WHERE with an explicit null check" + .to_string(), + )); + } + let target_var = target + .variable + .clone() + .unwrap_or_else(|| "_anon_1".to_string()); + let (min_hops, max_hops) = edge.var_length.unwrap_or((1, 1)); + ops.push(PhysicalOp::Expand { + source: first_var.clone(), + target: target_var.clone(), + edge_variable: edge.variable.clone(), + edge_types: edge.edge_types.clone(), + direction: edge.direction, + min_hops, + max_hops, + optional: true, + }); + bound.insert(target_var); + if let Some(evar) = &edge.variable { + bound.insert(evar.clone()); + } + } + Ok(()) +} + /// Compile a `MATCH p = shortestPath((a)-[*..N]-(b))` clause. /// /// Emits: NodeScan(a) -> NodeScan(b) -> ShortestPath(a, b). The executor @@ -644,6 +902,8 @@ fn push_node_scan(node: &PatternNode, var: &str, ops: &mut Vec) { variable: var.to_string(), label: node.labels.first().cloned(), prop_eq: node.properties.clone(), + prop_range: Vec::new(), + text_pred: Vec::new(), }); } else { ops.push(PhysicalOp::NodeScan { @@ -658,6 +918,168 @@ fn push_node_scan(node: &PatternNode, var: &str, ops: &mut Vec) { } } +/// Walk a WHERE expression's top-level AND-chain and push every +/// `var.prop literal|param` conjunct (either orientation) into the +/// scan op that BINDS `var` (upgrading a plain `NodeScan` to an +/// `IndexScan` when needed). W2-3. +/// +/// Soundness: a top-level AND conjunct must hold for every result row, so +/// restricting var's scan to a SUPERSET of rows satisfying it can never +/// drop a valid row — provided the executor's index lookup is a superset +/// of the residual Filter's semantics for that conjunct (see +/// `index_scan_keys`). Disjunctions (`OR`) are never extracted. +fn extract_range_conjuncts(expr: &Expr, ops: &mut Vec) { + match expr { + Expr::BinaryOp { + left, + op: BinaryOperator::And, + right, + } => { + extract_range_conjuncts(left, ops); + extract_range_conjuncts(right, ops); + } + Expr::BinaryOp { left, op, right } => { + let cmp = match op { + BinaryOperator::GreaterThan => RangeCmp::Gt, + BinaryOperator::GreaterEqual => RangeCmp::Gte, + BinaryOperator::LessThan => RangeCmp::Lt, + BinaryOperator::LessEqual => RangeCmp::Lte, + _ => return, + }; + // `var.prop value` or the mirrored `value var.prop`. + let (var, prop, value, cmp) = match (as_prop_access(left), as_prop_access(right)) { + (Some((var, prop)), None) if is_range_value(right) => { + (var, prop, (**right).clone(), cmp) + } + (None, Some((var, prop))) if is_range_value(left) => { + (var, prop, (**left).clone(), cmp.flipped()) + } + _ => return, + }; + // Find the scan that binds `var` and attach the conjunct. + for op in ops.iter_mut().rev() { + match op { + PhysicalOp::IndexScan { + variable, + prop_range, + .. + } if variable == var => { + prop_range.push((prop.to_owned(), cmp, value)); + return; + } + PhysicalOp::NodeScan { variable, label } if variable == var => { + *op = PhysicalOp::IndexScan { + variable: variable.clone(), + label: label.clone(), + prop_eq: Vec::new(), + prop_range: vec![(prop.to_owned(), cmp, value)], + text_pred: Vec::new(), + }; + return; + } + _ => {} + } + } + } + _ => {} + } +} + +/// Walk a WHERE expression's top-level AND-chain and push every +/// `var.prop literal|param` conjunct +/// into the scan op that BINDS `var` (upgrading a plain `NodeScan` to an +/// `IndexScan` when needed), mirroring `extract_range_conjuncts` (W2-3) for +/// text predicates (P3 design part B). +/// +/// Soundness: identical argument to `extract_range_conjuncts` — a top-level +/// AND conjunct must hold for every result row, so restricting `var`'s scan +/// to a SUPERSET of rows satisfying it can never drop a valid row. The +/// index probe (`SegmentTextIndex::candidate_rows`) is a presence-only +/// superset for ANY string predicate on that property — see `text_index.rs` +/// module docs for why token-level pruning is unsound here. Disjunctions +/// (`OR`) are never extracted. +fn extract_text_conjuncts(expr: &Expr, ops: &mut Vec) { + match expr { + Expr::BinaryOp { + left, + op: BinaryOperator::And, + right, + } => { + extract_text_conjuncts(left, ops); + extract_text_conjuncts(right, ops); + } + Expr::BinaryOp { + left, + op: text_op, + right, + } => { + if !matches!( + text_op, + BinaryOperator::Contains + | BinaryOperator::StartsWith + | BinaryOperator::EndsWith + | BinaryOperator::RegexMatch + ) { + return; + } + let Some((var, prop)) = as_prop_access(left) else { + return; + }; + if !is_text_value(right) { + return; + } + for op in ops.iter_mut().rev() { + match op { + PhysicalOp::IndexScan { + variable, + text_pred, + .. + } if variable == var => { + text_pred.push((prop.to_owned(), *text_op, (**right).clone())); + return; + } + PhysicalOp::NodeScan { variable, label } if variable == var => { + *op = PhysicalOp::IndexScan { + variable: variable.clone(), + label: label.clone(), + prop_eq: Vec::new(), + prop_range: Vec::new(), + text_pred: vec![(prop.to_owned(), *text_op, (**right).clone())], + }; + return; + } + _ => {} + } + } + } + _ => {} + } +} + +/// Is this expression usable as a text-predicate pattern (resolvable +/// without a row, mirroring `is_range_value`)? String literals and +/// parameters only. +fn is_text_value(expr: &Expr) -> bool { + matches!(expr, Expr::StringLit(_) | Expr::Parameter(_)) +} + +/// `n.prop` accessor on a plain variable, if the expression is one. +fn as_prop_access(expr: &Expr) -> Option<(&str, &str)> { + if let Expr::PropertyAccess { object, property } = expr { + if let Expr::Ident(var) = object.as_ref() { + return Some((var.as_str(), property.as_str())); + } + } + None +} + +/// Is this expression usable as an index range threshold (resolvable +/// without a row)? Numeric literals and parameters only — the executor +/// skips conjuncts whose parameter resolves to a non-number. +fn is_range_value(expr: &Expr) -> bool { + matches!(expr, Expr::Integer(_) | Expr::Float(_) | Expr::Parameter(_)) +} + /// Build a filter expression `var.k1 = v1 AND var.k2 = v2 ...` from a /// PatternNode's inline property map, expressed as a standalone Filter op. /// Used to apply inline node-property predicates `(v {k:e, …})` in both @@ -787,6 +1209,172 @@ mod tests { assert!(matches!(plan.operators[0], PhysicalOp::NodeScan { .. })); } + // ─── W2-3: WHERE range predicates drive the property index ──────────── + + #[test] + fn test_where_range_upgrades_scan_to_index_range() { + let query = parse_cypher(b"MATCH (n:Person) WHERE n.age > 30 RETURN n").expect("parse"); + let plan = compile(&query).expect("compile"); + match &plan.operators[0] { + PhysicalOp::IndexScan { + prop_eq, + prop_range, + .. + } => { + assert!(prop_eq.is_empty(), "no inline equality props"); + assert_eq!(prop_range.len(), 1, "one range conjunct"); + assert_eq!(prop_range[0].0, "age"); + assert_eq!(prop_range[0].1, RangeCmp::Gt); + } + other => panic!("expected range IndexScan, got {other:?}"), + } + // The full WHERE stays as a residual Filter (index is superset-only). + assert!( + plan.operators + .iter() + .any(|op| matches!(op, PhysicalOp::Filter { .. })), + "residual Filter must remain" + ); + } + + #[test] + fn test_where_range_conjuncts_compose_and_flip() { + let query = + parse_cypher(b"MATCH (n:N {id:3}) WHERE n.a >= 1 AND 10 > n.b AND n.c = 2 RETURN n") + .expect("parse"); + let plan = compile(&query).expect("compile"); + match &plan.operators[0] { + PhysicalOp::IndexScan { + prop_eq, + prop_range, + .. + } => { + assert_eq!(prop_eq.len(), 1, "inline {{id:3}} stays an eq conjunct"); + assert_eq!( + prop_range.len(), + 2, + "two range conjuncts, got {prop_range:?}" + ); + assert_eq!(prop_range[0].0, "a"); + assert_eq!(prop_range[0].1, RangeCmp::Gte); + // `10 > n.b` flips to b < 10. + assert_eq!(prop_range[1].0, "b"); + assert_eq!(prop_range[1].1, RangeCmp::Lt); + } + other => panic!("expected range IndexScan, got {other:?}"), + } + } + + #[test] + fn test_where_or_is_not_range_extracted() { + // Disjunctions cannot prune a scan (a row failing one branch may + // satisfy the other). + let query = parse_cypher(b"MATCH (n:N) WHERE n.a > 1 OR n.b < 2 RETURN n").expect("parse"); + let plan = compile(&query).expect("compile"); + assert!( + matches!(plan.operators[0], PhysicalOp::NodeScan { .. }), + "OR predicate must not upgrade the scan; ops = {:?}", + plan.operators + ); + } + + // ─── P3 design part B: WHERE text predicates drive the text index ───── + + #[test] + fn test_where_contains_upgrades_scan_to_index_text_pred() { + let query = + parse_cypher(b"MATCH (n:Person) WHERE n.bio CONTAINS 'rust' RETURN n").expect("parse"); + let plan = compile(&query).expect("compile"); + match &plan.operators[0] { + PhysicalOp::IndexScan { + prop_eq, + prop_range, + text_pred, + .. + } => { + assert!(prop_eq.is_empty()); + assert!(prop_range.is_empty()); + assert_eq!(text_pred.len(), 1, "one text conjunct, got {text_pred:?}"); + assert_eq!(text_pred[0].0, "bio"); + assert_eq!(text_pred[0].1, BinaryOperator::Contains); + } + other => panic!("expected text IndexScan, got {other:?}"), + } + // The full WHERE stays as a residual Filter (index is superset-only). + assert!( + plan.operators + .iter() + .any(|op| matches!(op, PhysicalOp::Filter { .. })), + "residual Filter must remain" + ); + } + + #[test] + fn test_where_starts_with_and_ends_with_upgrade_scan() { + for (query_bytes, expect_op) in [ + ( + b"MATCH (n:N) WHERE n.name STARTS WITH 'Al' RETURN n".as_slice(), + BinaryOperator::StartsWith, + ), + ( + b"MATCH (n:N) WHERE n.name ENDS WITH 'ce' RETURN n".as_slice(), + BinaryOperator::EndsWith, + ), + ( + b"MATCH (n:N) WHERE n.name =~ '.*x.*' RETURN n".as_slice(), + BinaryOperator::RegexMatch, + ), + ] { + let query = parse_cypher(query_bytes).expect("parse"); + let plan = compile(&query).expect("compile"); + match &plan.operators[0] { + PhysicalOp::IndexScan { text_pred, .. } => { + assert_eq!(text_pred.len(), 1, "ops = {:?}", plan.operators); + assert_eq!(text_pred[0].1, expect_op); + } + other => panic!( + "expected text IndexScan for {:?}, got {other:?}", + core::str::from_utf8(query_bytes) + ), + } + } + } + + #[test] + fn test_where_text_and_range_conjuncts_compose() { + // A single WHERE mixing a numeric range and a text predicate on + // DIFFERENT properties of the same variable must upgrade the SAME + // IndexScan with both hints populated. + let query = parse_cypher(b"MATCH (n:N) WHERE n.age > 30 AND n.bio CONTAINS 'x' RETURN n") + .expect("parse"); + let plan = compile(&query).expect("compile"); + match &plan.operators[0] { + PhysicalOp::IndexScan { + prop_range, + text_pred, + .. + } => { + assert_eq!(prop_range.len(), 1); + assert_eq!(text_pred.len(), 1); + assert_eq!(text_pred[0].0, "bio"); + } + other => panic!("expected combined IndexScan, got {other:?}"), + } + } + + #[test] + fn test_where_text_or_is_not_extracted() { + // Disjunctions cannot prune a scan, same rule as range conjuncts. + let query = + parse_cypher(b"MATCH (n:N) WHERE n.a CONTAINS 'x' OR n.b > 1 RETURN n").expect("parse"); + let plan = compile(&query).expect("compile"); + assert!( + matches!(plan.operators[0], PhysicalOp::NodeScan { .. }), + "OR predicate must not upgrade the scan; ops = {:?}", + plan.operators + ); + } + #[test] fn test_inline_prop_on_expanded_node_filters_after_expand() { // M2 — `MATCH (a {id:1})-[]->(b {id:3})`: a Filter after the NodeScan AND one after the Expand. @@ -880,26 +1468,38 @@ mod tests { assert!(cache.is_empty()); let plan = Arc::new(PhysicalPlan { operators: vec![] }); - cache.insert(42, plan.clone()); + cache.insert(42, plan.clone(), true); assert_eq!(cache.len(), 1); assert!(cache.get(42).is_some()); assert!(cache.get(99).is_none()); // Fill and evict - cache.insert(43, plan.clone()); - cache.insert(44, plan.clone()); + cache.insert(43, plan.clone(), true); + cache.insert(44, plan.clone(), true); assert_eq!(cache.len(), 2); } + #[test] + fn test_plan_cache_read_only_flag_round_trips() { + // W2-7: write plans are cached too; the flag tells the read path to + // treat them as misses and the write path to execute them directly. + let mut cache = PlanCache::new(4); + let plan = Arc::new(PhysicalPlan { operators: vec![] }); + cache.insert(1, plan.clone(), true); + cache.insert(2, plan.clone(), false); + assert_eq!(cache.get(1).map(|c| c.read_only), Some(true)); + assert_eq!(cache.get(2).map(|c| c.read_only), Some(false)); + } + #[test] fn test_plan_cache_lru_eviction_order() { let mut cache = PlanCache::new(2); let plan = Arc::new(PhysicalPlan { operators: vec![] }); - cache.insert(1, plan.clone()); - cache.insert(2, plan.clone()); + cache.insert(1, plan.clone(), true); + cache.insert(2, plan.clone(), true); // Touch 1 so 2 becomes the least-recently-used entry. assert!(cache.get(1).is_some()); - cache.insert(3, plan.clone()); + cache.insert(3, plan.clone(), true); assert!( cache.get(1).is_some(), "recently-used entry must survive eviction" @@ -927,7 +1527,13 @@ mod tests { let raw_hash = 100; let normalized_hash = 200; let auto_params = vec![("auto0".to_string(), Value::Int(42))]; - cache.insert_both(raw_hash, normalized_hash, plan.clone(), auto_params.clone()); + cache.insert_both( + raw_hash, + normalized_hash, + plan.clone(), + auto_params.clone(), + true, + ); assert_eq!(cache.len(), 2, "raw + normalized keys are distinct entries"); let raw_hit = cache.get(raw_hash).expect("raw-hash hit"); @@ -962,7 +1568,7 @@ mod tests { // raw_hash == normalized_hash -- must not create two entries. let mut cache = PlanCache::new(8); let plan = Arc::new(PhysicalPlan { operators: vec![] }); - cache.insert_both(50, 50, plan, Vec::new()); + cache.insert_both(50, 50, plan, Vec::new(), true); assert_eq!(cache.len(), 1); } @@ -986,6 +1592,7 @@ mod tests { normalized_hash, plan.clone(), vec![("auto0".to_string(), Value::Int(7))], + true, ); } assert_eq!(reparse_count, 1); @@ -1009,12 +1616,18 @@ mod tests { // in raw+normalized pairs. let mut cache = PlanCache::new(2); let plan = Arc::new(PhysicalPlan { operators: vec![] }); - cache.insert_both(1, 2, plan.clone(), vec![("a".to_string(), Value::Int(1))]); + cache.insert_both( + 1, + 2, + plan.clone(), + vec![("a".to_string(), Value::Int(1))], + true, + ); assert_eq!(cache.len(), 2); // Cache is now full (max_entries=2). A brand-new key must evict the // least-recently-used entry (raw key 1, since normalized key 2 was // inserted after it and is therefore more recently used). - cache.insert(3, plan); + cache.insert(3, plan, true); assert_eq!(cache.len(), 2); assert!( cache.get(1).is_none(), diff --git a/src/graph/cypher/result_cache.rs b/src/graph/cypher/result_cache.rs new file mode 100644 index 000000000..e4c161d25 --- /dev/null +++ b/src/graph/cypher/result_cache.rs @@ -0,0 +1,504 @@ +//! Cypher result cache: caches pre-encoded RESP reply bytes for read-only +//! `GRAPH.QUERY` / `GRAPH.RO_QUERY` results. +//! +//! # Design summary (Task #32 design doc, Part A) +//! +//! - Keyed by `(raw query-bytes hash, remaining-args hash)` — mirrors +//! `PlanCache`'s raw-hash pre-lookup (`planner::hash_query`), so +//! literal-per-query text (e.g. the bench's `{id: }` point queries) +//! naturally gets distinct entries without a separate normalization pass. +//! - Invalidated by `NamedGraph::write_gen`, a per-graph monotonic counter +//! bumped at every mutation site that changes query-visible state (see +//! `NamedGraph::touch`). A cache hit whose `write_gen_at_insert` doesn't +//! match the graph's current `write_gen` is treated as a miss — the stale +//! entry is left in place and naturally overwritten by the next `put()` +//! for that key (same lazy-eviction philosophy as `PlanCache`). +//! - Stores wire bytes (`bytes::Bytes`), not a `Frame` tree: `FrameVec` is a +//! *boxed* `SmallVec` (`src/protocol/frame.rs`), so cloning a cached +//! `Frame::Array` re-allocates the box/smallvec on every hit. Caching the +//! RESP-encoded `Bytes` instead makes a hit an O(1) atomic refcount bump +//! (`Bytes::clone()`), replayed via the pre-existing `Frame::PreSerialized` +//! passthrough variant (already used by the GET fast path) — no new +//! protocol-layer variant needed. +//! - RESP2 and RESP3 clients must never share a slot: `encode_frame` +//! branches on `protocol_version` (arrays/maps/booleans/doubles serialize +//! differently). Each entry carries two independently-populated slots. +//! A slot is populated lazily, from whichever protocol version first +//! `put()`s that key — the OTHER slot for an already-cached key is simply +//! a cache miss (not an auto re-encode: this cache never retains the +//! source `ExecResult`, only the wire bytes, so there is nothing to +//! re-encode from). This is a deliberate simplification vs. a design that +//! would keep the `ExecResult` alive to serve either protocol from one +//! `put` — it trades a slightly lower hit rate under mixed RESP2/RESP3 +//! traffic against the *same* query text for a strictly smaller resident +//! footprint (exactly the trade-off caching wire bytes over Frame trees +//! is for in the first place). A miss is always safe: the caller +//! re-executes and re-populates. +//! +//! # NOT covered by this cache (deliberately -- see call sites) +//! +//! - `GRAPH.EXPLAIN` / `GRAPH.PROFILE` never consult or populate it (they +//! already bypass `PlanCache` for the same reason: stable-output debug +//! surfaces, not a query-result semantics to cache). +//! - `--decay` queries are wall-clock-dependent (`TemporalDecayScorer::now`) +//! and are never cached regardless of `write_gen` — two decay queries at +//! different real times against an *unchanged* graph legitimately differ. +//! - Only `Ok` executions are cached; a `TIMEOUT`-truncated or any other +//! `Err` result never reaches `put()` — gating on `Result::Ok` alone +//! excludes every truncated-result class without a separate flag. + +use bytes::Bytes; +use std::sync::OnceLock; + +use crate::graph::fasthash::FxHashMap; + +/// Cache key: raw Cypher query-bytes hash + remaining-args hash. +/// +/// `query_hash` mirrors `planner::hash_query` applied to the exact client +/// query bytes (not the literal-normalized text) — every distinct literal +/// value naturally gets its own key, which is exactly what a *result* cache +/// wants (a plan cache wants to SHARE across literals; a result cache must +/// NOT, since the results differ). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ResultCacheKey { + pub query_hash: u64, + pub args_hash: u64, +} + +/// One cached query result: up to two lazily-populated wire-byte slots +/// (RESP2 / RESP3), the write-generation this entry was computed against, +/// and an LRU access tick. +struct ResultCacheEntry { + resp2: OnceLock, + resp3: OnceLock, + write_gen_at_insert: u64, + used: u64, +} + +impl ResultCacheEntry { + fn resident_bytes(&self) -> usize { + self.resp2.get().map_or(0, Bytes::len) + self.resp3.get().map_or(0, Bytes::len) + } +} + +/// Bounded, per-graph LRU cache of pre-encoded Cypher query reply bytes. +/// +/// Wrapped in `parking_lot::Mutex` on `NamedGraph` (same interior-mutability +/// pattern as `plan_cache`) so read handlers reachable through an immutable +/// `&NamedGraph` borrow (`graph_query_readonly` never takes `&mut +/// GraphStore`) can still record hits/insert entries. +pub struct ResultCache { + entries: FxHashMap, + max_entries: usize, + max_bytes: usize, + cur_bytes: usize, + /// Monotonic access counter backing LRU eviction (mirrors `PlanCache`). + tick: u64, + /// Doorkeeper admission filter (TinyLFU-style, one-shot): direct-mapped + /// table of key fingerprints. A key is admitted to the cache only on its + /// SECOND sighting — the first records the fingerprint and declines — + /// so scan/cycle access patterns whose reuse distance exceeds capacity + /// never pay the miss cost (RESP serialization + insert + O(n) LRU + /// eviction scan; measured −21% qps on a 5K-key cycling workload + /// without this gate). Collisions are benign in both directions: a + /// false admit merely reverts to pre-doorkeeper behavior for that key; + /// an overwritten fingerprint merely delays that key's admission. + doorkeeper: Vec, +} + +/// Doorkeeper slots (power of two; 2048 × 8 B = 16 KiB per graph). +const DOORKEEPER_SLOTS: usize = 2048; + +impl ResultCache { + /// Create a new result cache. `max_entries` bounds distinct + /// `(query, args)` keys; `max_bytes` bounds the sum of resident RESP2 + + /// RESP3 wire bytes across all entries (a soft cap -- enforced after + /// each `put`, so a single oversized insert can transiently exceed it + /// by that one entry's size before eviction catches up). + pub fn new(max_entries: usize, max_bytes: usize) -> Self { + Self { + entries: FxHashMap::default(), + max_entries, + max_bytes, + cur_bytes: 0, + tick: 0, + doorkeeper: vec![0; DOORKEEPER_SLOTS], + } + } + + /// Admission check — call BEFORE serializing a reply for `put()`, so a + /// declined first sighting pays nothing beyond this probe. + /// + /// Returns `true` (admit) when the key is already cached (either + /// protocol slot, fresh or stale — a re-`put` refreshes it) or when its + /// doorkeeper fingerprint was recorded by a previous sighting. Returns + /// `false` on a first sighting, recording the fingerprint. + pub fn should_admit(&mut self, key: ResultCacheKey) -> bool { + if self.entries.contains_key(&key) { + return true; + } + // Mix both hash halves so the slot index and the stored fingerprint + // use different bit ranges of the key. + let fp = key.query_hash ^ key.args_hash.rotate_left(32); + // Reserve 0 as "empty" — remap a genuinely-zero fingerprint. + let fp = if fp == 0 { 1 } else { fp }; + #[allow(clippy::cast_possible_truncation)] + let slot = (fp as usize) & (DOORKEEPER_SLOTS - 1); + if self.doorkeeper[slot] == fp { + return true; + } + self.doorkeeper[slot] = fp; + false + } + + /// Look up a cached reply for `key` at the caller's negotiated + /// `protocol_version` (2 or 3; anything `>= 3` is treated as RESP3). + /// + /// Returns `None` when: no entry exists, the entry is stale (its + /// `write_gen_at_insert` no longer matches `current_write_gen` -- + /// left in place, not evicted eagerly, exactly like a `PlanCache` miss), + /// or the requested protocol's slot hasn't been populated yet for this + /// key. Every `None` path is a safe, ordinary cache miss -- callers + /// re-execute and `put()` the fresh result. + pub fn get( + &mut self, + key: ResultCacheKey, + current_write_gen: u64, + protocol_version: u8, + ) -> Option { + self.tick += 1; + let tick = self.tick; + let entry = self.entries.get_mut(&key)?; + if entry.write_gen_at_insert != current_write_gen { + return None; + } + entry.used = tick; + let slot = if protocol_version >= 3 { + &entry.resp3 + } else { + &entry.resp2 + }; + slot.get().cloned() + } + + /// Insert (or update) the wire-encoded reply for `key` at + /// `protocol_version`, stamped with `write_gen` (the graph's write + /// generation *captured before the query ran* -- see `NamedGraph::touch` + /// docs for why this ordering matters for the race guard). + /// + /// A key whose existing entry is stale (`write_gen_at_insert` doesn't + /// match `write_gen`) is replaced wholesale -- both protocol slots reset + /// -- rather than patched in place: a stale RESP2 slot must never survive + /// next to a freshly-inserted RESP3 slot under the same key. + pub fn put(&mut self, key: ResultCacheKey, write_gen: u64, protocol_version: u8, bytes: Bytes) { + self.tick += 1; + let tick = self.tick; + + if let Some(existing) = self.entries.get(&key) { + if existing.write_gen_at_insert != write_gen { + self.cur_bytes -= existing.resident_bytes(); + self.entries.remove(&key); + } + } + + if !self.entries.contains_key(&key) && self.entries.len() >= self.max_entries { + self.evict_lru(); + } + + let entry = self.entries.entry(key).or_insert_with(|| ResultCacheEntry { + resp2: OnceLock::new(), + resp3: OnceLock::new(), + write_gen_at_insert: write_gen, + used: tick, + }); + entry.used = tick; + entry.write_gen_at_insert = write_gen; + + let len = bytes.len(); + let slot = if protocol_version >= 3 { + &entry.resp3 + } else { + &entry.resp2 + }; + // `set` only fails if the slot is already populated (a second + // `put()` for the same key+protocol before invalidation, e.g. a + // benign race between two concurrent identical queries on the same + // shard thread -- can't happen cross-thread since NamedGraph is + // shard-owned, but a single-threaded re-entrant `put` for the exact + // same key is still a harmless no-op here). + if slot.set(bytes).is_ok() { + self.cur_bytes += len; + } + + self.enforce_byte_budget(); + } + + /// Evict least-recently-used entries until `cur_bytes <= max_bytes`. + /// Stops at one remaining entry so a single reply larger than + /// `max_bytes` doesn't spin the cache empty on every insert. + fn enforce_byte_budget(&mut self) { + while self.cur_bytes > self.max_bytes && self.entries.len() > 1 { + self.evict_lru(); + } + } + + /// Remove the least-recently-used entry (by `used` tick), if any. + fn evict_lru(&mut self) { + let Some(lru_key) = self + .entries + .iter() + .min_by_key(|(_, e)| e.used) + .map(|(k, _)| *k) + else { + return; + }; + if let Some(evicted) = self.entries.remove(&lru_key) { + self.cur_bytes -= evicted.resident_bytes(); + } + } + + /// Number of distinct `(query, args)` keys currently cached. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether the cache is empty. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Clear all cached entries (`FLUSHALL` / `FLUSHDB` / `GRAPH.DELETE`). + pub fn clear(&mut self) { + self.entries.clear(); + self.cur_bytes = 0; + } + + /// Resident bytes: wire-byte payloads plus a per-entry key/bookkeeping + /// estimate, so the shard's elastic memory budget sees this cache the + /// same way it already sees `SegmentPropertyIndexes` / `GraphHnsw` + /// (`csr/storage.rs::resident_bytes` doc comment -- an unaccounted + /// lazily-built structure is exactly the class of bug those precedents + /// fixed; this cache must not repeat it). + pub fn resident_bytes(&self) -> usize { + self.cur_bytes + + self.entries.len() + * (std::mem::size_of::() + std::mem::size_of::()) + } +} + +/// Default entry-count cap: modest by design (per-graph, not shard-wide) -- +/// a handful of hundred distinct hot queries covers the skewed-access +/// production case the design doc identifies as the real justification +/// (A.5 "Case 2"), without the cache itself becoming a memory-budget risk +/// on graphs with many named indexes. +pub const DEFAULT_MAX_ENTRIES: usize = 256; + +/// Default byte budget: ~4 MiB. A 1-hop/2-hop point-query reply is tens to +/// low-hundreds of bytes RESP-encoded, so this comfortably covers +/// `DEFAULT_MAX_ENTRIES` entries at both protocol versions with headroom. +pub const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; + +#[cfg(test)] +mod tests { + use super::*; + + fn key(q: u64, a: u64) -> ResultCacheKey { + ResultCacheKey { + query_hash: q, + args_hash: a, + } + } + + #[test] + fn test_miss_on_empty_cache() { + let mut cache = ResultCache::new(8, 1024); + assert_eq!(cache.get(key(1, 1), 0, 2), None); + } + + #[test] + fn test_doorkeeper_declines_first_sighting_admits_second() { + let mut cache = ResultCache::new(8, 1024); + assert!( + !cache.should_admit(key(1, 1)), + "first sighting must decline" + ); + assert!(cache.should_admit(key(1, 1)), "second sighting must admit"); + // Still admitted on subsequent sightings. + assert!(cache.should_admit(key(1, 1))); + } + + #[test] + fn test_doorkeeper_always_admits_already_cached_key() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"a")); + // Cached (even without a prior should_admit sighting): admit — + // covers the refresh-after-invalidation and second-protocol-slot + // paths. + assert!(cache.should_admit(key(1, 1))); + } + + #[test] + fn test_doorkeeper_cycling_scan_never_admits() { + // A scan whose reuse distance exceeds the doorkeeper table: every + // slot is overwritten before its key returns, so nothing is ever + // admitted and the cache stays empty (the −21% regression guard). + let mut cache = ResultCache::new(8, 1024); + for round in 0..2u64 { + let mut admitted = 0; + for i in 0..(super::DOORKEEPER_SLOTS as u64 * 4) { + if cache.should_admit(key(i, i)) { + admitted += 1; + } + } + // Direct-mapped fingerprint collisions can admit a stray key; + // the point is the overwhelming majority never get in. + assert!( + admitted < (super::DOORKEEPER_SLOTS as u64) / 16, + "cycling scan round {round} admitted {admitted} keys" + ); + } + assert!( + cache.is_empty(), + "no put() ever ran — cache must stay empty" + ); + } + + #[test] + fn test_put_then_get_hit_same_protocol() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 5, 2, Bytes::from_static(b"*1\r\n:1\r\n")); + assert_eq!( + cache.get(key(1, 1), 5, 2), + Some(Bytes::from_static(b"*1\r\n:1\r\n")) + ); + } + + #[test] + fn test_get_miss_on_write_gen_mismatch() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 5, 2, Bytes::from_static(b"data")); + // Graph mutated since insert (write_gen advanced) -- stale, must miss. + assert_eq!(cache.get(key(1, 1), 6, 2), None); + } + + #[test] + fn test_different_args_hash_is_different_key() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"a")); + cache.put(key(1, 2), 0, 2, Bytes::from_static(b"b")); + assert_eq!(cache.get(key(1, 1), 0, 2), Some(Bytes::from_static(b"a"))); + assert_eq!(cache.get(key(1, 2), 0, 2), Some(Bytes::from_static(b"b"))); + assert_eq!(cache.len(), 2); + } + + #[test] + fn test_protocol_version_isolation() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"resp2-bytes")); + // Same key, RESP3 slot not yet populated -- must miss, not + // accidentally return the RESP2 bytes. + assert_eq!(cache.get(key(1, 1), 0, 3), None); + cache.put(key(1, 1), 0, 3, Bytes::from_static(b"resp3-bytes")); + assert_eq!( + cache.get(key(1, 1), 0, 2), + Some(Bytes::from_static(b"resp2-bytes")) + ); + assert_eq!( + cache.get(key(1, 1), 0, 3), + Some(Bytes::from_static(b"resp3-bytes")) + ); + } + + #[test] + fn test_stale_entry_replaced_resets_both_protocol_slots() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"old-resp2")); + cache.put(key(1, 1), 0, 3, Bytes::from_static(b"old-resp3")); + // Mutation advances write_gen; a fresh RESP2-only put must not leave + // the OLD resp3 bytes reachable under the new generation. + cache.put(key(1, 1), 1, 2, Bytes::from_static(b"new-resp2")); + assert_eq!( + cache.get(key(1, 1), 1, 2), + Some(Bytes::from_static(b"new-resp2")) + ); + assert_eq!( + cache.get(key(1, 1), 1, 3), + None, + "stale resp3 slot must not survive a write_gen bump" + ); + } + + #[test] + fn test_lru_eviction_order() { + let mut cache = ResultCache::new(2, usize::MAX); + cache.put(key(1, 0), 0, 2, Bytes::from_static(b"a")); + cache.put(key(2, 0), 0, 2, Bytes::from_static(b"b")); + // Touch key(1,0) so key(2,0) becomes LRU. + assert!(cache.get(key(1, 0), 0, 2).is_some()); + cache.put(key(3, 0), 0, 2, Bytes::from_static(b"c")); + assert_eq!(cache.len(), 2); + assert_eq!( + cache.get(key(2, 0), 0, 2), + None, + "key(2,0) was LRU and must have been evicted" + ); + assert!(cache.get(key(1, 0), 0, 2).is_some()); + assert!(cache.get(key(3, 0), 0, 2).is_some()); + } + + #[test] + fn test_byte_budget_eviction() { + // `max_bytes` bounds the wire-payload total (`cur_bytes`); each + // payload here is 100 bytes, so a 350-byte budget fits at most 3. + let mut cache = ResultCache::new(usize::MAX, 350); + let payload = Bytes::from(vec![0u8; 100]); + for i in 0..5u64 { + cache.put(key(i, 0), 0, 2, payload.clone()); + } + assert!( + cache.len() < 5, + "byte budget must have evicted at least one entry before max_entries was hit" + ); + // `resident_bytes()` additionally reports fixed per-entry + // bookkeeping overhead (key + entry struct) on top of the raw wire + // payload that `max_bytes` governs -- assert the payload-only + // component (`len() * 100`) stays within budget, not the + // overhead-inclusive total. + assert!( + cache.len() * 100 <= 350, + "wire-payload bytes must be within budget: {} entries * 100", + cache.len() + ); + } + + #[test] + fn test_resident_bytes_tracks_puts_and_clear() { + let mut cache = ResultCache::new(8, 1024); + assert_eq!(cache.resident_bytes(), 0); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"12345")); + assert!(cache.resident_bytes() >= 5); + cache.clear(); + assert_eq!(cache.resident_bytes(), 0); + } + + #[test] + fn test_clear_empties_cache() { + let mut cache = ResultCache::new(8, 1024); + cache.put(key(1, 1), 0, 2, Bytes::from_static(b"x")); + assert!(!cache.is_empty()); + cache.clear(); + assert!(cache.is_empty()); + assert_eq!(cache.get(key(1, 1), 0, 2), None); + } + + #[test] + fn test_single_oversized_entry_does_not_starve_cache_empty() { + let mut cache = ResultCache::new(8, 10); + cache.put(key(1, 1), 0, 2, Bytes::from(vec![0u8; 100])); + // Budget is 10 bytes but the entry is 100 -- must still be retained + // (the "stop at one remaining entry" guard), not evicted into an + // infinite loop or a permanently-empty cache. + assert_eq!(cache.len(), 1); + assert!(cache.get(key(1, 1), 0, 2).is_some()); + } +} diff --git a/src/graph/hnsw_bridge.rs b/src/graph/hnsw_bridge.rs index 53aa0a11f..88bdeba1d 100644 --- a/src/graph/hnsw_bridge.rs +++ b/src/graph/hnsw_bridge.rs @@ -53,6 +53,11 @@ pub struct GraphHnsw { /// Flat unit-normalized embeddings, bridge-id-major. vecs: Vec, dim: usize, + /// CSR rows that HAVE an embedding but are not indexed (dimension + /// mismatch with the majority dim, or degenerate norm). Callers doing a + /// whole-segment bridge scan (HYB-04) must score these exactly — the + /// list makes that possible without re-scanning every row. + uncovered: Vec, } impl core::fmt::Debug for GraphHnsw { @@ -76,6 +81,7 @@ impl GraphHnsw { let node_count = seg.node_count(); let mut rows: Vec = Vec::new(); let mut vecs: Vec = Vec::new(); + let mut uncovered: Vec = Vec::new(); let mut dim = 0usize; for row in 0..node_count { @@ -83,16 +89,19 @@ impl GraphHnsw { continue; }; if emb.is_empty() { + uncovered.push(row as u32); continue; } if dim == 0 { dim = emb.len(); } if emb.len() != dim { + uncovered.push(row as u32); continue; } let norm: f32 = emb.iter().map(|x| x * x).sum::().sqrt(); if !(norm.is_finite() && norm > 0.0) { + uncovered.push(row as u32); continue; } for x in &mut emb { @@ -139,6 +148,7 @@ impl GraphHnsw { row_to_id, vecs, dim, + uncovered, }) } @@ -179,6 +189,13 @@ impl GraphHnsw { self.graph.resident_bytes() + rows_bytes + row_to_id_bytes + vecs_bytes } + /// CSR rows with an embedding the bridge could not index (dim mismatch + /// / degenerate norm). Whole-segment bridge scans score these exactly. + #[inline] + pub fn uncovered_embedded_rows(&self) -> &[u32] { + &self.uncovered + } + #[inline] fn vec_of(&self, id: u32) -> &[f32] { &self.vecs[id as usize * self.dim..(id as usize + 1) * self.dim] @@ -473,4 +490,82 @@ mod tests { ); } } + + #[test] + fn test_bridge_tracks_uncovered_embedded_rows() { + // Mixed embeddings: majority dim-8, a few dim-4 + one zero-norm. + // The bridge indexes the dim-8 rows and must LIST the rest so a + // whole-segment bridge scan (HYB-04) can score them exactly. + let mut g = MemGraph::new(1_000_000); + let mut keys = Vec::new(); + for i in 0..20u64 { + keys.push(g.add_node( + smallvec![0u16], + smallvec::SmallVec::new(), + Some(embedding(i, 8)), + 1, + )); + } + for i in 20..23u64 { + keys.push(g.add_node( + smallvec![0u16], + smallvec::SmallVec::new(), + Some(embedding(i, 4)), // dim mismatch -> uncovered + 1, + )); + } + keys.push(g.add_node( + smallvec![0u16], + smallvec::SmallVec::new(), + Some(vec![0.0; 8]), // zero norm -> uncovered + 1, + )); + for i in 0..keys.len() - 1 { + g.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + let seg = CsrStorage::from( + CsrSegment::from_frozen(g.freeze().expect("freeze"), 10).expect("csr"), + ); + let bridge = GraphHnsw::build(&seg, 1).expect("bridge"); + assert_eq!(bridge.len(), 20); + assert_eq!(bridge.uncovered_embedded_rows().len(), 4); + // Covered and uncovered partition the embedded rows exactly. + for &row in bridge.uncovered_embedded_rows() { + assert!(!bridge.contains_row(row)); + assert!(seg.node_embedding(row).is_some()); + } + let embedded = (0..seg.node_count()) + .filter(|&r| seg.node_embedding(r).is_some()) + .count(); + assert_eq!( + bridge.len() + bridge.uncovered_embedded_rows().len(), + embedded + ); + } + + #[test] + fn test_bridge_builds_off_thread() { + // The production accessor must NOT block the calling (shard event + // loop) thread on the HNSW construction: the first call kicks off a + // background build and reports "not ready"; the result installs + // asynchronously. 4200 embeddings clears BRIDGE_MIN_VECTORS. + let (seg, _) = frozen_segment(4200, 8); + let seg = std::sync::Arc::new(seg); + assert!( + seg.hnsw_bridge().is_none(), + "first call must return immediately (build runs off-thread)" + ); + let mut installed = false; + for _ in 0..1200 { + if seg.hnsw_bridge().is_some() { + installed = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(installed, "background bridge build never installed"); + let bridge = seg.hnsw_bridge().expect("installed"); + assert_eq!(bridge.len(), 4200); + } } diff --git a/src/graph/hybrid.rs b/src/graph/hybrid/mod.rs similarity index 65% rename from src/graph/hybrid.rs rename to src/graph/hybrid/mod.rs index f150caeeb..f03d58247 100644 --- a/src/graph/hybrid.rs +++ b/src/graph/hybrid/mod.rs @@ -117,545 +117,11 @@ pub fn select_strategy(candidate_count: usize, threshold: usize) -> FilterStrate } } -// --------------------------------------------------------------------------- -// HYB-01: Graph-filtered vector search -// --------------------------------------------------------------------------- - -/// Configuration for graph-filtered vector search. -pub struct GraphFilteredSearch { - /// Start node for graph traversal. - pub start_node: NodeKey, - /// Maximum traversal depth (hops). - pub hops: u32, - /// Optional edge type filter. - pub edge_type_filter: Option, - /// Query vector for similarity scoring. - pub query_vector: Vec, - /// Number of top results to return. - pub k: usize, - /// Strategy selection threshold (default 10K). - pub threshold: usize, - /// Maximum frontier size to prevent OOM. - pub frontier_cap: usize, -} - -impl GraphFilteredSearch { - /// Create with defaults (threshold=10K, frontier_cap=100K). - pub fn new(start_node: NodeKey, hops: u32, query_vector: Vec, k: usize) -> Self { - Self { - start_node, - hops, - edge_type_filter: None, - query_vector, - k, - threshold: DEFAULT_STRATEGY_THRESHOLD, - frontier_cap: 100_000, - } - } - - /// Execute graph-filtered vector search. - /// - /// 1. BFS N hops from start_node -> collect candidate NodeKeys - /// 2. Auto-select strategy (brute-force vs pre-filter) - /// 3. Score candidates by cosine similarity to query_vector - /// 4. Return top-K results - pub fn execute( - &self, - memgraph: &MemGraph, - csr_segs: &[Arc], - lsn: u64, - ) -> Result, HybridError> { - if self.query_vector.is_empty() { - return Err(HybridError::EmptyQueryVector); - } - - let view = MergedNodeView::new(memgraph, csr_segs); - - // Verify start node exists in EITHER tier. - if !view.contains(self.start_node) { - return Err(HybridError::NodeNotFound); - } - - // Step 1: BFS to collect candidates with graph distance. - let candidates = bfs_collect( - memgraph, - csr_segs, - self.start_node, - self.hops, - self.edge_type_filter, - self.frontier_cap, - lsn, - )?; - - // Step 2: Select strategy. - let strategy = select_strategy(candidates.len(), self.threshold); - - // Step 3: Score candidates. HnswPreFilter routes CSR-resident - // candidates through each segment's HNSW bridge and leaves the - // rest (mutable tier, bridge-less rows, dim mismatch) in - // `residual` for exact scoring below. - let mut scored: Vec = Vec::with_capacity(candidates.len().min(4096)); - let residual: Vec<(NodeKey, u32)> = match strategy { - FilterStrategy::BruteForce => candidates, - FilterStrategy::HnswPreFilter => hnsw_prefilter_score( - memgraph, - csr_segs, - candidates, - &self.query_vector, - self.k, - &mut scored, - ), - }; - - // Exact scoring for whatever the pre-filter did not cover - // (everything, under BruteForce): embedding resolved from the - // mutable tier or the CSR v5 blob. - for (node_key, graph_dist) in &residual { - let Some(embedding) = view.embedding(*node_key) else { - continue; // Skip nodes without embeddings. - }; - - let sim = simd::cosine_similarity(&embedding, &self.query_vector); - scored.push(HybridResult { - node: *node_key, - score: sim, - graph_distance: Some(*graph_dist), - context: Vec::new(), - }); - } - - // Step 4: Sort descending by score, take top-K. - scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); - scored.truncate(self.k); - - Ok(scored) - } -} +mod search; +mod walk_rerank; -/// Route candidates through per-segment HNSW bridges (HnswPreFilter). -/// -/// Partitions `candidates` into per-segment groups (a candidate joins a -/// group when its segment has a bridge covering its row at the query's -/// dimension) and searches each bridge with an allow-filter over the -/// group's rows. Bridge hits are pushed to `scored` with their EXACT -/// cosine (bridge vectors are unit-normalized copies of the originals). -/// Returns the residual candidates for exact scoring: mutable-tier nodes, -/// bridge-less rows, dim mismatches, and any whole group whose beam -/// under-filled k (approximate-search rescue — never silently truncate). -fn hnsw_prefilter_score( - memgraph: &MemGraph, - csr_segs: &[Arc], - candidates: Vec<(NodeKey, u32)>, - query: &[f32], - k: usize, - scored: &mut Vec, -) -> Vec<(NodeKey, u32)> { - use crate::graph::fasthash::FxHashMap; - - let mut residual: Vec<(NodeKey, u32)> = Vec::new(); - let mut groups: Vec> = - (0..csr_segs.len()).map(|_| FxHashMap::default()).collect(); - - 'cand: for (key, gdist) in candidates { - // Mutable tier wins (same precedence as MergedNodeView::embedding). - if memgraph.get_node(key).is_some() { - residual.push((key, gdist)); - continue; - } - for (i, seg) in csr_segs.iter().enumerate() { - if let Some(row) = seg.lookup_node(key) { - if let Some(bridge) = seg.hnsw_bridge() { - if bridge.dim() == query.len() && bridge.contains_row(row) { - groups[i].insert(row, (key, gdist)); - continue 'cand; - } - } - // Resident here but not bridge-searchable: score exactly. - residual.push((key, gdist)); - continue 'cand; - } - } - residual.push((key, gdist)); - } - - for (i, group) in groups.iter().enumerate() { - if group.is_empty() { - continue; - } - let Some(bridge) = csr_segs[i].hnsw_bridge() else { - // Unreachable by construction; stay exact if it ever isn't. - residual.extend(group.values().copied()); - continue; - }; - let hits = bridge.search(query, k, |row| group.contains_key(&row)); - if hits.len() < k.min(group.len()) { - // Beam under-filled the ask: rescue with exact scoring. - residual.extend(group.values().copied()); - continue; - } - for (row, sim) in hits { - if let Some(&(key, gdist)) = group.get(&row) { - scored.push(HybridResult { - node: key, - score: sim, - graph_distance: Some(gdist), - context: Vec::new(), - }); - } - } - } - - residual -} - -// --------------------------------------------------------------------------- -// HYB-02: Vector-to-graph expansion -// --------------------------------------------------------------------------- - -/// Configuration for vector-to-graph expansion. -pub struct VectorToGraphExpansion { - /// Query vector for initial similarity search. - pub query_vector: Vec, - /// Number of top vector results before expansion. - pub k: usize, - /// Expansion depth (hops from each result). - pub expansion_hops: u32, - /// Optional edge type filter for expansion. - pub edge_type_filter: Option, -} - -impl VectorToGraphExpansion { - /// Create with defaults. - pub fn new(query_vector: Vec, k: usize, expansion_hops: u32) -> Self { - Self { - query_vector, - k, - expansion_hops, - edge_type_filter: None, - } - } - - /// Execute vector-to-graph expansion. - /// - /// 1. Brute-force search all nodes with embeddings for top-K by similarity - /// 2. For each result, BFS expand N hops for context - /// 3. Return results with context neighbors - /// - /// `candidate_nodes` is a pre-collected list of all node keys to search. - /// The caller should provide this (e.g., from MemGraph iteration or label index). - pub fn execute( - &self, - memgraph: &MemGraph, - csr_segs: &[Arc], - candidate_nodes: &[NodeKey], - lsn: u64, - ) -> Result, HybridError> { - if self.query_vector.is_empty() { - return Err(HybridError::EmptyQueryVector); - } - - let view = MergedNodeView::new(memgraph, csr_segs); - let committed = roaring::RoaringBitmap::new(); - - // Step 1: Score all candidate nodes by cosine similarity. - let mut scored: Vec<(NodeKey, f64)> = Vec::with_capacity(candidate_nodes.len()); - - for &node_key in candidate_nodes { - if !view.is_visible(node_key, 0, 0, &committed, None) { - continue; - } - let Some(embedding) = view.embedding(node_key) else { - continue; - }; - - let sim = simd::cosine_similarity(&embedding, &self.query_vector); - scored.push((node_key, sim)); - } - - // Top-K by similarity. - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)); - scored.truncate(self.k); - - // Step 2: Expand each result by N hops. - let mut results: Vec = Vec::with_capacity(scored.len()); - - for (node_key, score) in scored { - let context = if self.expansion_hops > 0 { - collect_context( - memgraph, - csr_segs, - node_key, - self.expansion_hops, - self.edge_type_filter, - lsn, - ) - } else { - Vec::new() - }; - - results.push(HybridResult { - node: node_key, - score, - graph_distance: None, - context, - }); - } - - Ok(results) - } -} - -// --------------------------------------------------------------------------- -// HYB-03: Vector-guided walk (beam search) -// --------------------------------------------------------------------------- - -/// Configuration for vector-guided graph walk. -pub struct VectorGuidedWalk { - /// Seed node to start the walk. - pub seed_node: NodeKey, - /// Query vector: walk toward neighbors most similar to this. - pub query_vector: Vec, - /// Maximum walk depth. - pub max_depth: u32, - /// Beam width: how many candidates to expand at each step. - pub beam_width: usize, - /// Minimum similarity threshold: stop walking if best neighbor is below this. - pub min_similarity: f64, -} - -impl VectorGuidedWalk { - /// Create with defaults (beam_width=5, min_similarity=0.0). - pub fn new(seed_node: NodeKey, query_vector: Vec, max_depth: u32) -> Self { - Self { - seed_node, - query_vector, - max_depth, - beam_width: 5, - min_similarity: 0.0, - } - } - - /// Execute vector-guided walk. - /// - /// At each step, expand all neighbors of the current beam, score by cosine - /// similarity, and keep the top `beam_width` for the next step. Returns the - /// walk path: all visited nodes with their cumulative scores. - pub fn execute( - &self, - memgraph: &MemGraph, - csr_segs: &[Arc], - lsn: u64, - ) -> Result, HybridError> { - if self.query_vector.is_empty() { - return Err(HybridError::EmptyQueryVector); - } - - let view = MergedNodeView::new(memgraph, csr_segs); - let reader = SegmentMergeReader::new(Some(memgraph), csr_segs, Direction::Both, lsn, None); - - if !view.contains(self.seed_node) { - return Err(HybridError::NodeNotFound); - } - - let mut visited: HashSet = HashSet::new(); - visited.insert(self.seed_node); - - // Score the seed node. - let seed_score = view - .embedding(self.seed_node) - .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) - .unwrap_or(0.0); - - let mut results: Vec = Vec::new(); - results.push(HybridResult { - node: self.seed_node, - score: seed_score, - graph_distance: Some(0), - context: Vec::new(), - }); - - // Current beam: (node_key, score). - let mut beam: Vec<(NodeKey, f64)> = vec![(self.seed_node, seed_score)]; - - for depth in 1..=self.max_depth { - let mut candidates: Vec<(NodeKey, f64)> = Vec::new(); - - for &(current, _) in &beam { - // Expand neighbors across both tiers. - for merged in reader.neighbors(current) { - let neighbor_key = merged.node; - if visited.contains(&neighbor_key) { - continue; - } - - let sim = view - .embedding(neighbor_key) - .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) - .unwrap_or(0.0); - - candidates.push((neighbor_key, sim)); - } - } - - if candidates.is_empty() { - break; // No more unvisited neighbors. - } - - // Sort by similarity descending, take top beam_width. - candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)); - candidates.truncate(self.beam_width); - - // Check minimum similarity threshold. - let best_sim = candidates.first().map(|c| c.1).unwrap_or(0.0); - if best_sim < self.min_similarity { - break; // Best candidate below threshold. - } - - // Add to results and prepare next beam. - beam.clear(); - for (node_key, sim) in &candidates { - if visited.insert(*node_key) { - results.push(HybridResult { - node: *node_key, - score: *sim, - graph_distance: Some(depth), - context: Vec::new(), - }); - beam.push((*node_key, *sim)); - } - } - - if beam.is_empty() { - break; - } - } - - Ok(results) - } -} - -// --------------------------------------------------------------------------- -// HYB-04: Graph-constrained re-ranking -// --------------------------------------------------------------------------- - -/// Configuration for graph-constrained re-ranking. -/// -/// Re-ranks ALL nodes with embeddings by a combined score: -/// `alpha * vector_score + (1 - alpha) * 1 / (1 + graph_distance)` -/// -/// Graph distances are computed via a single batch BFS from the reference node, -/// giving O(frontier) cost instead of O(k * frontier) for per-candidate BFS. -/// Nodes not reachable within `max_hops` receive a penalty distance of -/// `max_hops + 1`. -pub struct GraphConstrainedReRanker { - /// Reference node for graph distance computation. - pub reference_node: NodeKey, - /// Maximum BFS depth for distance computation. - pub max_hops: u32, - /// Weight for vector similarity score (0.0 = graph only, 1.0 = vector only). - pub alpha: f64, - /// Number of top results to return. - pub k: usize, - /// Query vector for cosine similarity scoring. - pub query_vector: Vec, - /// Maximum frontier size for BFS to prevent OOM. - pub frontier_cap: usize, -} - -impl GraphConstrainedReRanker { - /// Create with defaults (frontier_cap=100K). - pub fn new( - reference_node: NodeKey, - max_hops: u32, - alpha: f64, - query_vector: Vec, - k: usize, - ) -> Self { - Self { - reference_node, - max_hops, - alpha, - k, - query_vector, - frontier_cap: 100_000, - } - } - - /// Execute graph-constrained re-ranking. - /// - /// 1. Validate inputs (reference node exists, non-empty vector, alpha in range). - /// 2. Single batch BFS from reference node to compute graph distances. - /// 3. Iterate ALL nodes with embeddings, compute cosine similarity. - /// 4. Combine: `alpha * vector_score + (1-alpha) * 1/(1+graph_dist)`. - /// 5. Sort descending, return top-K. - pub fn execute( - &self, - memgraph: &MemGraph, - csr_segs: &[Arc], - lsn: u64, - ) -> Result, HybridError> { - if self.query_vector.is_empty() { - return Err(HybridError::EmptyQueryVector); - } - - let view = MergedNodeView::new(memgraph, csr_segs); - - // Validate reference node exists in EITHER tier. - if !view.contains(self.reference_node) { - return Err(HybridError::NodeNotFound); - } - - // Clamp alpha to [0.0, 1.0]. - let alpha = self.alpha.clamp(0.0, 1.0); - let penalty_dist = self.max_hops + 1; - - // Step 1: Single batch BFS from reference node — O(frontier). - let bfs_results = bfs_collect( - memgraph, - csr_segs, - self.reference_node, - self.max_hops, - None, - self.frontier_cap, - lsn, - )?; - - // Build O(1) distance lookup map. - let mut distance_map: HashMap = HashMap::with_capacity(bfs_results.len() + 1); - distance_map.insert(self.reference_node, 0); - for (node_key, dist) in &bfs_results { - distance_map.insert(*node_key, *dist); - } - - // Step 2: Score ALL nodes with embeddings, across both tiers. - let committed = roaring::RoaringBitmap::new(); - let mut scored: Vec = Vec::with_capacity(distance_map.len()); - - view.for_each_visible_node(None, 0, 0, &committed, None, |node_key| { - let Some(embedding) = view.embedding(node_key) else { - return; // Skip nodes without embeddings. - }; - - let vector_score = simd::cosine_similarity(&embedding, &self.query_vector); - let graph_dist = distance_map.get(&node_key).copied().unwrap_or(penalty_dist); - let graph_score = 1.0 / (1.0 + graph_dist as f64); - let combined = alpha * vector_score + (1.0 - alpha) * graph_score; - - scored.push(HybridResult { - node: node_key, - score: combined, - graph_distance: Some(graph_dist), - context: Vec::new(), - }); - }); - - // Step 3: Sort descending by combined score, take top-K. - scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); - scored.truncate(self.k); - - Ok(scored) - } -} +pub use search::*; +pub use walk_rerank::*; // --------------------------------------------------------------------------- // BFS collect helper (shared by HYB-01, HYB-04) @@ -1476,4 +942,100 @@ mod tests { "prefilter must prune" ); } + + #[test] + fn test_expansion_bridge_matches_brute_force() { + // HYB-02 (W2-5): at >= threshold candidates the expansion routes + // frozen rows through the segment bridge; scores stay REAL cosines + // and the top-k closely tracks exact scoring. + let (mg, seg, _center) = frozen_star_segment(); + assert!(seg.hnsw_bridge_for_test().is_some(), "bridge must build"); + + let candidates: Vec = seg + .node_meta() + .iter() + .map(|m| slotmap::KeyData::from_ffi(m.external_id).into()) + .collect(); + let query = det_embedding(31337, 8); + let segs = vec![seg.clone()]; + + let mut bridged = VectorToGraphExpansion::new(query.clone(), 5, 1); + bridged.threshold = 1; // force HnswPreFilter + let mut brute = VectorToGraphExpansion::new(query.clone(), 5, 1); + brute.threshold = usize::MAX; // force BruteForce + + let bres = bridged + .execute(&mg, &segs, &candidates, u64::MAX - 1) + .expect("bridged ok"); + let xres = brute + .execute(&mg, &segs, &candidates, u64::MAX - 1) + .expect("brute ok"); + assert_eq!(bres.len(), 5); + assert_eq!(xres.len(), 5); + + let view = MergedNodeView::new(&mg, &segs); + for r in &bres { + let emb = view.embedding(r.node).expect("embedding"); + let exact = simd::cosine_similarity(&emb, &query); + assert!((r.score - exact).abs() < 1e-4); + assert_eq!(r.graph_distance, None, "HYB-02 has no graph distance"); + assert!(!r.context.is_empty(), "expansion context must populate"); + } + let brute_set: HashSet = xres.iter().map(|r| r.node).collect(); + let overlap = bres.iter().filter(|r| brute_set.contains(&r.node)).count(); + assert!(overlap >= 4, "HYB-02 bridge/brute overlap {overlap}/5"); + } + + #[test] + fn test_rerank_bridge_matches_brute_force() { + // HYB-04 (W2-5): non-frontier nodes share the penalty graph term, + // so the bridged path answers with the segment's top-k by cosine; + // frontier + mutable nodes stay exact. Every returned score must + // still be the true combined formula. + let (mut mg, seg, center) = frozen_star_segment(); + assert!(seg.hnsw_bridge_for_test().is_some(), "bridge must build"); + + // A mutable-tier node too (must be scored exactly, never dropped). + let hot = mg.add_node(smallvec![0], empty_props(), Some(det_embedding(500, 8)), 20); + + let query = det_embedding(4711, 8); + let segs = vec![seg.clone()]; + + let mut bridged = GraphConstrainedReRanker::new(center, 1, 0.7, query.clone(), 8); + bridged.threshold = 1; // force the bridge path + let mut brute = GraphConstrainedReRanker::new(center, 1, 0.7, query.clone(), 8); + brute.threshold = usize::MAX; // full exact scan + + let bres = bridged.execute(&mg, &segs, u64::MAX - 1).expect("bridged"); + let xres = brute.execute(&mg, &segs, u64::MAX - 1).expect("brute"); + assert_eq!(bres.len(), 8); + assert_eq!(xres.len(), 8); + + // Bridged scores must be the exact combined formula for that node. + let view = MergedNodeView::new(&mg, &segs); + for r in &bres { + let emb = view.embedding(r.node).expect("embedding"); + let cos = simd::cosine_similarity(&emb, &query); + let gd = r.graph_distance.expect("distance always set") as f64; + let expect = 0.7 * cos + 0.3 * (1.0 / (1.0 + gd)); + assert!( + (r.score - expect).abs() < 1e-4, + "node {:?}: {} vs {expect}", + r.node, + r.score + ); + } + // The star is 1-hop from center: every spoke is IN the frontier, so + // bridge and brute must agree BIT-EXACTLY on the frontier class; the + // mutable node rides along in both. + let brute_set: HashSet = xres.iter().map(|r| r.node).collect(); + let overlap = bres.iter().filter(|r| brute_set.contains(&r.node)).count(); + assert!(overlap >= 7, "HYB-04 bridge/brute overlap {overlap}/8"); + let hot_in_bridged = bres.iter().any(|r| r.node == hot); + let hot_in_brute = xres.iter().any(|r| r.node == hot); + assert_eq!( + hot_in_bridged, hot_in_brute, + "mutable-tier node must rank identically in both paths" + ); + } } diff --git a/src/graph/hybrid/search.rs b/src/graph/hybrid/search.rs new file mode 100644 index 000000000..7d094ba7e --- /dev/null +++ b/src/graph/hybrid/search.rs @@ -0,0 +1,313 @@ +//! HYB-01 graph-filtered vector search + HYB-02 vector-to-graph +//! expansion (split from hybrid.rs per the 1500-line module rule). + +use super::*; + +// --------------------------------------------------------------------------- +// HYB-01: Graph-filtered vector search +// --------------------------------------------------------------------------- + +/// Configuration for graph-filtered vector search. +pub struct GraphFilteredSearch { + /// Start node for graph traversal. + pub start_node: NodeKey, + /// Maximum traversal depth (hops). + pub hops: u32, + /// Optional edge type filter. + pub edge_type_filter: Option, + /// Query vector for similarity scoring. + pub query_vector: Vec, + /// Number of top results to return. + pub k: usize, + /// Strategy selection threshold (default 10K). + pub threshold: usize, + /// Maximum frontier size to prevent OOM. + pub frontier_cap: usize, +} + +impl GraphFilteredSearch { + /// Create with defaults (threshold=10K, frontier_cap=100K). + pub fn new(start_node: NodeKey, hops: u32, query_vector: Vec, k: usize) -> Self { + Self { + start_node, + hops, + edge_type_filter: None, + query_vector, + k, + threshold: DEFAULT_STRATEGY_THRESHOLD, + frontier_cap: 100_000, + } + } + + /// Execute graph-filtered vector search. + /// + /// 1. BFS N hops from start_node -> collect candidate NodeKeys + /// 2. Auto-select strategy (brute-force vs pre-filter) + /// 3. Score candidates by cosine similarity to query_vector + /// 4. Return top-K results + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { + if self.query_vector.is_empty() { + return Err(HybridError::EmptyQueryVector); + } + + let view = MergedNodeView::new(memgraph, csr_segs); + + // Verify start node exists in EITHER tier. + if !view.contains(self.start_node) { + return Err(HybridError::NodeNotFound); + } + + // Step 1: BFS to collect candidates with graph distance. + let candidates = bfs_collect( + memgraph, + csr_segs, + self.start_node, + self.hops, + self.edge_type_filter, + self.frontier_cap, + lsn, + )?; + + // Step 2: Select strategy. + let strategy = select_strategy(candidates.len(), self.threshold); + + // Step 3: Score candidates. HnswPreFilter routes CSR-resident + // candidates through each segment's HNSW bridge and leaves the + // rest (mutable tier, bridge-less rows, dim mismatch) in + // `residual` for exact scoring below. + let mut scored: Vec = Vec::with_capacity(candidates.len().min(4096)); + let residual: Vec<(NodeKey, u32)> = match strategy { + FilterStrategy::BruteForce => candidates, + FilterStrategy::HnswPreFilter => hnsw_prefilter_score( + memgraph, + csr_segs, + candidates, + &self.query_vector, + self.k, + &mut scored, + ), + }; + + // Exact scoring for whatever the pre-filter did not cover + // (everything, under BruteForce): embedding resolved from the + // mutable tier or the CSR v5 blob. + for (node_key, graph_dist) in &residual { + let Some(embedding) = view.embedding(*node_key) else { + continue; // Skip nodes without embeddings. + }; + + let sim = simd::cosine_similarity(&embedding, &self.query_vector); + scored.push(HybridResult { + node: *node_key, + score: sim, + graph_distance: Some(*graph_dist), + context: Vec::new(), + }); + } + + // Step 4: Sort descending by score, take top-K. + scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); + scored.truncate(self.k); + + Ok(scored) + } +} + +/// Route candidates through per-segment HNSW bridges (HnswPreFilter). +/// +/// Partitions `candidates` into per-segment groups (a candidate joins a +/// group when its segment has a bridge covering its row at the query's +/// dimension) and searches each bridge with an allow-filter over the +/// group's rows. Bridge hits are pushed to `scored` with their EXACT +/// cosine (bridge vectors are unit-normalized copies of the originals). +/// Returns the residual candidates for exact scoring: mutable-tier nodes, +/// bridge-less rows, dim mismatches, and any whole group whose beam +/// under-filled k (approximate-search rescue — never silently truncate). +pub(super) fn hnsw_prefilter_score( + memgraph: &MemGraph, + csr_segs: &[Arc], + candidates: Vec<(NodeKey, u32)>, + query: &[f32], + k: usize, + scored: &mut Vec, +) -> Vec<(NodeKey, u32)> { + use crate::graph::fasthash::FxHashMap; + + let mut residual: Vec<(NodeKey, u32)> = Vec::new(); + let mut groups: Vec> = + (0..csr_segs.len()).map(|_| FxHashMap::default()).collect(); + + 'cand: for (key, gdist) in candidates { + // Mutable tier wins (same precedence as MergedNodeView::embedding). + if memgraph.get_node(key).is_some() { + residual.push((key, gdist)); + continue; + } + for (i, seg) in csr_segs.iter().enumerate() { + if let Some(row) = seg.lookup_node(key) { + if let Some(bridge) = seg.hnsw_bridge() { + if bridge.dim() == query.len() && bridge.contains_row(row) { + groups[i].insert(row, (key, gdist)); + continue 'cand; + } + } + // Resident here but not bridge-searchable: score exactly. + residual.push((key, gdist)); + continue 'cand; + } + } + residual.push((key, gdist)); + } + + for (i, group) in groups.iter().enumerate() { + if group.is_empty() { + continue; + } + let Some(bridge) = csr_segs[i].hnsw_bridge() else { + // Unreachable by construction; stay exact if it ever isn't. + residual.extend(group.values().copied()); + continue; + }; + let hits = bridge.search(query, k, |row| group.contains_key(&row)); + if hits.len() < k.min(group.len()) { + // Beam under-filled the ask: rescue with exact scoring. + residual.extend(group.values().copied()); + continue; + } + for (row, sim) in hits { + if let Some(&(key, gdist)) = group.get(&row) { + scored.push(HybridResult { + node: key, + score: sim, + graph_distance: Some(gdist), + context: Vec::new(), + }); + } + } + } + + residual +} + +// --------------------------------------------------------------------------- +// HYB-02: Vector-to-graph expansion +// --------------------------------------------------------------------------- + +/// Configuration for vector-to-graph expansion. +pub struct VectorToGraphExpansion { + /// Query vector for initial similarity search. + pub query_vector: Vec, + /// Number of top vector results before expansion. + pub k: usize, + /// Expansion depth (hops from each result). + pub expansion_hops: u32, + /// Optional edge type filter for expansion. + pub edge_type_filter: Option, + /// Strategy selection threshold (default 10K): candidate sets at or + /// above it route CSR-resident nodes through the per-segment HNSW + /// bridge (W2-5), exactly like HYB-01's HnswPreFilter. + pub threshold: usize, +} + +impl VectorToGraphExpansion { + /// Create with defaults. + pub fn new(query_vector: Vec, k: usize, expansion_hops: u32) -> Self { + Self { + query_vector, + k, + expansion_hops, + edge_type_filter: None, + threshold: DEFAULT_STRATEGY_THRESHOLD, + } + } + + /// Execute vector-to-graph expansion. + /// + /// 1. Top-K candidates by cosine similarity — exact scoring for small + /// sets; at >= `threshold` candidates, CSR-resident nodes route + /// through the per-segment HNSW bridge (mutable-tier and + /// bridge-less rows stay exact, under-filled beams rescue to exact). + /// 2. For each result, BFS expand N hops for context + /// 3. Return results with context neighbors + /// + /// `candidate_nodes` is a pre-collected list of all node keys to search. + /// The caller should provide this (e.g., from MemGraph iteration or label index). + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + candidate_nodes: &[NodeKey], + lsn: u64, + ) -> Result, HybridError> { + if self.query_vector.is_empty() { + return Err(HybridError::EmptyQueryVector); + } + + let view = MergedNodeView::new(memgraph, csr_segs); + let committed = roaring::RoaringBitmap::new(); + + // Step 1: visibility-filter, then score. The prefilter helper wants + // (key, graph_distance) pairs; HYB-02 has no graph distance, so 0 + // stands in and the field is cleared on the final results. + let visible: Vec<(NodeKey, u32)> = candidate_nodes + .iter() + .copied() + .filter(|&k| view.is_visible(k, 0, 0, &committed, None)) + .map(|k| (k, 0)) + .collect(); + + let mut scored: Vec = Vec::with_capacity(visible.len().min(4096)); + let residual: Vec<(NodeKey, u32)> = match select_strategy(visible.len(), self.threshold) { + FilterStrategy::BruteForce => visible, + FilterStrategy::HnswPreFilter => hnsw_prefilter_score( + memgraph, + csr_segs, + visible, + &self.query_vector, + self.k, + &mut scored, + ), + }; + + for (node_key, _) in &residual { + let Some(embedding) = view.embedding(*node_key) else { + continue; + }; + let sim = simd::cosine_similarity(&embedding, &self.query_vector); + scored.push(HybridResult { + node: *node_key, + score: sim, + graph_distance: None, + context: Vec::new(), + }); + } + + // Top-K by similarity. + scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); + scored.truncate(self.k); + + // Step 2: Expand each result by N hops. + for result in &mut scored { + result.graph_distance = None; + result.context = if self.expansion_hops > 0 { + collect_context( + memgraph, + csr_segs, + result.node, + self.expansion_hops, + self.edge_type_filter, + lsn, + ) + } else { + Vec::new() + }; + } + + Ok(scored) + } +} diff --git a/src/graph/hybrid/walk_rerank.rs b/src/graph/hybrid/walk_rerank.rs new file mode 100644 index 000000000..b1a908412 --- /dev/null +++ b/src/graph/hybrid/walk_rerank.rs @@ -0,0 +1,399 @@ +//! HYB-03 vector-guided walk + HYB-04 graph-constrained re-ranking +//! (split from hybrid.rs per the 1500-line module rule). + +use super::*; + +// --------------------------------------------------------------------------- +// HYB-03: Vector-guided walk (beam search) +// --------------------------------------------------------------------------- + +/// Configuration for vector-guided graph walk. +pub struct VectorGuidedWalk { + /// Seed node to start the walk. + pub seed_node: NodeKey, + /// Query vector: walk toward neighbors most similar to this. + pub query_vector: Vec, + /// Maximum walk depth. + pub max_depth: u32, + /// Beam width: how many candidates to expand at each step. + pub beam_width: usize, + /// Minimum similarity threshold: stop walking if best neighbor is below this. + pub min_similarity: f64, +} + +impl VectorGuidedWalk { + /// Create with defaults (beam_width=5, min_similarity=0.0). + pub fn new(seed_node: NodeKey, query_vector: Vec, max_depth: u32) -> Self { + Self { + seed_node, + query_vector, + max_depth, + beam_width: 5, + min_similarity: 0.0, + } + } + + /// Execute vector-guided walk. + /// + /// At each step, expand all neighbors of the current beam, score by cosine + /// similarity, and keep the top `beam_width` for the next step. Returns the + /// walk path: all visited nodes with their cumulative scores. + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { + if self.query_vector.is_empty() { + return Err(HybridError::EmptyQueryVector); + } + + let view = MergedNodeView::new(memgraph, csr_segs); + let reader = SegmentMergeReader::new(Some(memgraph), csr_segs, Direction::Both, lsn, None); + + if !view.contains(self.seed_node) { + return Err(HybridError::NodeNotFound); + } + + let mut visited: HashSet = HashSet::new(); + visited.insert(self.seed_node); + + // Score the seed node. + let seed_score = view + .embedding(self.seed_node) + .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) + .unwrap_or(0.0); + + let mut results: Vec = Vec::new(); + results.push(HybridResult { + node: self.seed_node, + score: seed_score, + graph_distance: Some(0), + context: Vec::new(), + }); + + // Current beam: (node_key, score). + let mut beam: Vec<(NodeKey, f64)> = vec![(self.seed_node, seed_score)]; + + for depth in 1..=self.max_depth { + let mut candidates: Vec<(NodeKey, f64)> = Vec::new(); + + for &(current, _) in &beam { + // Expand neighbors across both tiers. + for merged in reader.neighbors(current) { + let neighbor_key = merged.node; + if visited.contains(&neighbor_key) { + continue; + } + + let sim = view + .embedding(neighbor_key) + .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) + .unwrap_or(0.0); + + candidates.push((neighbor_key, sim)); + } + } + + if candidates.is_empty() { + break; // No more unvisited neighbors. + } + + // Sort by similarity descending, take top beam_width. + candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)); + candidates.truncate(self.beam_width); + + // Check minimum similarity threshold. + let best_sim = candidates.first().map(|c| c.1).unwrap_or(0.0); + if best_sim < self.min_similarity { + break; // Best candidate below threshold. + } + + // Add to results and prepare next beam. + beam.clear(); + for (node_key, sim) in &candidates { + if visited.insert(*node_key) { + results.push(HybridResult { + node: *node_key, + score: *sim, + graph_distance: Some(depth), + context: Vec::new(), + }); + beam.push((*node_key, *sim)); + } + } + + if beam.is_empty() { + break; + } + } + + Ok(results) + } +} + +// --------------------------------------------------------------------------- +// HYB-04: Graph-constrained re-ranking +// --------------------------------------------------------------------------- + +/// Configuration for graph-constrained re-ranking. +/// +/// Re-ranks ALL nodes with embeddings by a combined score: +/// `alpha * vector_score + (1 - alpha) * 1 / (1 + graph_distance)` +/// +/// Graph distances are computed via a single batch BFS from the reference node, +/// giving O(frontier) cost instead of O(k * frontier) for per-candidate BFS. +/// Nodes not reachable within `max_hops` receive a penalty distance of +/// `max_hops + 1`. +pub struct GraphConstrainedReRanker { + /// Reference node for graph distance computation. + pub reference_node: NodeKey, + /// Maximum BFS depth for distance computation. + pub max_hops: u32, + /// Weight for vector similarity score (0.0 = graph only, 1.0 = vector only). + pub alpha: f64, + /// Number of top results to return. + pub k: usize, + /// Query vector for cosine similarity scoring. + pub query_vector: Vec, + /// Maximum frontier size for BFS to prevent OOM. + pub frontier_cap: usize, + /// Strategy selection threshold (default 10K): at or above this many + /// total nodes, frozen segments answer through their HNSW bridge (W2-5) + /// instead of a full scan. + pub threshold: usize, +} + +impl GraphConstrainedReRanker { + /// Create with defaults (frontier_cap=100K). + pub fn new( + reference_node: NodeKey, + max_hops: u32, + alpha: f64, + query_vector: Vec, + k: usize, + ) -> Self { + Self { + reference_node, + max_hops, + alpha, + k, + query_vector, + frontier_cap: 100_000, + threshold: DEFAULT_STRATEGY_THRESHOLD, + } + } + + /// Execute graph-constrained re-ranking. + /// + /// 1. Validate inputs (reference node exists, non-empty vector, alpha in range). + /// 2. Single batch BFS from reference node to compute graph distances. + /// 3. Iterate ALL nodes with embeddings, compute cosine similarity. + /// 4. Combine: `alpha * vector_score + (1-alpha) * 1/(1+graph_dist)`. + /// 5. Sort descending, return top-K. + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { + if self.query_vector.is_empty() { + return Err(HybridError::EmptyQueryVector); + } + + let view = MergedNodeView::new(memgraph, csr_segs); + + // Validate reference node exists in EITHER tier. + if !view.contains(self.reference_node) { + return Err(HybridError::NodeNotFound); + } + + // Clamp alpha to [0.0, 1.0]. + let alpha = self.alpha.clamp(0.0, 1.0); + let penalty_dist = self.max_hops + 1; + + // Step 1: Single batch BFS from reference node — O(frontier). + let bfs_results = bfs_collect( + memgraph, + csr_segs, + self.reference_node, + self.max_hops, + None, + self.frontier_cap, + lsn, + )?; + + // Build O(1) distance lookup map. + let mut distance_map: HashMap = HashMap::with_capacity(bfs_results.len() + 1); + distance_map.insert(self.reference_node, 0); + for (node_key, dist) in &bfs_results { + distance_map.insert(*node_key, *dist); + } + + // Step 2: score nodes with embeddings across both tiers. Every node + // OUTSIDE the BFS frontier shares the same graph term (the penalty + // distance), so within that class the combined ranking is monotone + // in cosine — which lets a large frozen segment answer with its + // HNSW bridge's top-k instead of a full scan (W2-5). Frontier nodes + // (bounded by frontier_cap) and the mutable tier (bounded by + // edge_threshold) are always scored exactly. + let committed = roaring::RoaringBitmap::new(); + let mut scored: Vec = Vec::with_capacity(distance_map.len()); + + let total_nodes: usize = memgraph.node_count() + + csr_segs + .iter() + .map(|s| s.node_count() as usize) + .sum::(); + let use_bridge = matches!( + select_strategy(total_nodes, self.threshold), + FilterStrategy::HnswPreFilter + ); + + if !use_bridge { + view.for_each_visible_node(None, 0, 0, &committed, None, |node_key| { + let Some(embedding) = view.embedding(node_key) else { + return; // Skip nodes without embeddings. + }; + + let vector_score = simd::cosine_similarity(&embedding, &self.query_vector); + let graph_dist = distance_map.get(&node_key).copied().unwrap_or(penalty_dist); + let graph_score = 1.0 / (1.0 + graph_dist as f64); + let combined = alpha * vector_score + (1.0 - alpha) * graph_score; + + scored.push(HybridResult { + node: node_key, + score: combined, + graph_distance: Some(graph_dist), + context: Vec::new(), + }); + }); + } else { + let push = |scored: &mut Vec, node_key: NodeKey, vector_score: f64| { + let graph_dist = distance_map.get(&node_key).copied().unwrap_or(penalty_dist); + let graph_score = 1.0 / (1.0 + graph_dist as f64); + scored.push(HybridResult { + node: node_key, + score: alpha * vector_score + (1.0 - alpha) * graph_score, + graph_distance: Some(graph_dist), + context: Vec::new(), + }); + }; + + // Mutable tier: exact. + for (key, node) in memgraph.iter_nodes() { + if !crate::graph::visibility::is_node_visible(node, 0, 0, &committed, None) { + continue; + } + let Some(embedding) = view.embedding(key) else { + continue; + }; + push( + &mut scored, + key, + simd::cosine_similarity(&embedding, &self.query_vector), + ); + } + + for (i, seg) in csr_segs.iter().enumerate() { + // Row filter shared by every path below: MVCC-visible, not + // shadowed by a mutable copy-up (W2-2), and the NEWEST + // resident copy across segments (list is newest-first). + let row_admissible = |row: u32| -> Option { + let meta = seg.node_meta().get(row as usize)?; + if !crate::graph::visibility::is_meta_visible(meta, 0, 0, &committed, None) { + return None; + } + let key = NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id)); + if memgraph.get_node(key).is_some() { + return None; + } + if csr_segs[..i] + .iter() + .any(|newer| newer.lookup_node(key).is_some()) + { + return None; + } + Some(key) + }; + let exact_row = |scored: &mut Vec, row: u32, key: NodeKey| { + if let Some(emb) = seg.node_embedding(row) { + push( + scored, + key, + simd::cosine_similarity(&emb, &self.query_vector), + ); + } + }; + + let bridge = seg + .hnsw_bridge() + .filter(|b| b.dim() == self.query_vector.len()); + let Some(bridge) = bridge else { + // No usable bridge (not built yet / too small / dim + // mismatch): exact scan of this segment. + for row in 0..seg.node_count() as u32 { + if let Some(key) = row_admissible(row) { + exact_row(&mut scored, row, key); + } + } + continue; + }; + + // (a) Frontier rows resident in this segment: exact — + // O(frontier) lookups, not a row scan. + for &frontier_key in distance_map.keys() { + let Some(row) = seg.lookup_node(frontier_key) else { + continue; + }; + if row_admissible(row) == Some(frontier_key) { + exact_row(&mut scored, row, frontier_key); + } + } + + // (b) Embedded rows the bridge could not index: exact. + for &row in bridge.uncovered_embedded_rows() { + let Some(key) = row_admissible(row) else { + continue; + }; + if !distance_map.contains_key(&key) { + exact_row(&mut scored, row, key); + } + } + + // (c) The rest: bridge top-k among non-frontier rows. + let hits = bridge.search(&self.query_vector, self.k, |row| { + row_admissible(row).is_some_and(|key| !distance_map.contains_key(&key)) + }); + if hits.len() < self.k.min(bridge.len()) { + // Beam under-filled the ask (or the allowed set is just + // small): rescue with an exact scan of the covered + // non-frontier rows — never silently truncate. + for row in 0..seg.node_count() as u32 { + if !bridge.contains_row(row) { + continue; // unembedded, or already exact in (b) + } + let Some(key) = row_admissible(row) else { + continue; + }; + if !distance_map.contains_key(&key) { + exact_row(&mut scored, row, key); + } + } + continue; + } + for (row, sim) in hits { + if let Some(key) = row_admissible(row) { + push(&mut scored, key, sim); + } + } + } + } + + // Step 3: Sort descending by combined score, take top-K. + scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); + scored.truncate(self.k); + + Ok(scored) + } +} diff --git a/src/graph/index.rs b/src/graph/index.rs index ffd759770..ff0003399 100644 --- a/src/graph/index.rs +++ b/src/graph/index.rs @@ -11,7 +11,26 @@ use ordered_float::OrderedFloat; use roaring::RoaringBitmap; use smallvec::SmallVec; -use crate::graph::types::{EdgeMeta, NodeKey, NodeMeta}; +use crate::graph::fasthash::FxHashMap; +use crate::graph::types::{EdgeMeta, NodeKey, NodeMeta, PropertyMap, PropertyValue}; + +/// Normalize a property value into f64 numeric space: `Int`/`Float` as-is, +/// `Bool` as 0.0/1.0. `String`/`Bytes` return `None` (indexed by xxh64 hash +/// instead — see [`SegmentPropertyIndexes`] / [`MutablePropertyIndex`]). +/// +/// Single source of truth for "what counts as numerically equal" shared by +/// BOTH the frozen tier (`SegmentPropertyIndexes::build`/`rows_eq`) and the +/// mutable tier (`MutablePropertyIndex`) so the two tiers can never silently +/// drift on equality semantics. +#[inline] +fn normalize_numeric(value: &PropertyValue) -> Option { + match value { + PropertyValue::Int(i) => Some(*i as f64), + PropertyValue::Float(f) => Some(*f), + PropertyValue::Bool(b) => Some(u8::from(*b) as f64), + PropertyValue::String(_) | PropertyValue::Bytes(_) => None, + } +} // --------------------------------------------------------------------------- // LabelIndex @@ -302,6 +321,16 @@ impl PropertyIndex { result } + /// Less-than-or-equal query: returns rows where property <= threshold. + pub fn lte(&self, threshold: f64) -> RoaringBitmap { + let key = OrderedFloat(threshold); + let mut result = RoaringBitmap::new(); + for (_, bitmap) in self.tree.range(..=key) { + result |= bitmap; + } + result + } + /// Number of distinct values in the index. pub fn distinct_values(&self) -> usize { self.tree.len() @@ -360,7 +389,6 @@ pub struct SegmentPropertyIndexes { impl SegmentPropertyIndexes { /// Build from CSR node metadata + the v5 node-property blob. pub fn build(node_meta: &[NodeMeta], node_props_blob: &[u8]) -> Self { - use crate::graph::types::PropertyValue; let mut numeric: HashMap = HashMap::new(); let mut strings: HashMap> = HashMap::new(); for (row, nm) in node_meta.iter().enumerate() { @@ -370,13 +398,7 @@ impl SegmentPropertyIndexes { let props = crate::graph::csr::props::decode_node_props(node_props_blob, nm.property_offset); for (pid, val) in &props { - let num = match val { - PropertyValue::Int(i) => Some(*i as f64), - PropertyValue::Float(f) => Some(*f), - PropertyValue::Bool(b) => Some(u8::from(*b) as f64), - PropertyValue::String(_) | PropertyValue::Bytes(_) => None, - }; - if let Some(v) = num { + if let Some(v) = normalize_numeric(val) { numeric .entry(*pid) .or_insert_with(|| PropertyIndex::new(*pid)) @@ -398,22 +420,20 @@ impl SegmentPropertyIndexes { /// Rows whose property `prop_id` equals `value` (superset semantics for /// hashed strings / Bool-Int aliasing — see type docs). - pub fn rows_eq( - &self, - prop_id: u16, - value: &crate::graph::types::PropertyValue, - ) -> RoaringBitmap { - use crate::graph::types::PropertyValue; - match value { - PropertyValue::Int(i) => self.numeric_eq(prop_id, *i as f64), - PropertyValue::Float(f) => self.numeric_eq(prop_id, *f), - PropertyValue::Bool(b) => self.numeric_eq(prop_id, u8::from(*b) as f64), - PropertyValue::String(s) | PropertyValue::Bytes(s) => self - .strings - .get(&prop_id) - .and_then(|m| m.get(&xxhash_rust::xxh64::xxh64(s, 0))) - .cloned() - .unwrap_or_default(), + pub fn rows_eq(&self, prop_id: u16, value: &PropertyValue) -> RoaringBitmap { + match normalize_numeric(value) { + Some(v) => self.numeric_eq(prop_id, v), + None => match value { + PropertyValue::String(s) | PropertyValue::Bytes(s) => self + .strings + .get(&prop_id) + .and_then(|m| m.get(&xxhash_rust::xxh64::xxh64(s, 0))) + .cloned() + .unwrap_or_default(), + PropertyValue::Int(_) | PropertyValue::Float(_) | PropertyValue::Bool(_) => { + RoaringBitmap::new() + } + }, } } @@ -451,6 +471,13 @@ impl SegmentPropertyIndexes { numeric + strings } + /// The numeric B-tree for `prop_id`, if any row indexed a numeric value + /// under it. `None` means no row can satisfy a numeric range on this + /// property (the build is exhaustive — see type docs). + pub fn numeric_index(&self, prop_id: u16) -> Option<&PropertyIndex> { + self.numeric.get(&prop_id) + } + fn numeric_eq(&self, prop_id: u16, v: f64) -> RoaringBitmap { self.numeric .get(&prop_id) @@ -459,6 +486,212 @@ impl SegmentPropertyIndexes { } } +// --------------------------------------------------------------------------- +// MutablePropertyIndex +// --------------------------------------------------------------------------- + +/// Incrementally-maintained property index over `MemGraph`'s mutable tier. +/// +/// Mirrors [`SegmentPropertyIndexes`]'s numeric-BTree / string-hash split, +/// but: +/// - keys by [`NodeKey`] directly (the mutable tier has no stable dense +/// row space — NodeKeys are slotmap ffi-encoded, sparse, and reused +/// only within a generation; reusing a `RoaringBitmap`-of-row scheme +/// here would reintroduce an ABA hazard the generational key design +/// exists to prevent — see the mutable-property-index design doc, +/// rejected alternatives). +/// - is maintained INCREMENTALLY on every write (insert/remove) instead +/// of built once, exhaustively, at freeze time (the mutable tier is, by +/// definition, still being written to). +/// - is NOT keyed by label, same as `SegmentPropertyIndexes` — label is +/// applied as a separate check at query time. +/// - has EXACT (not superset) removal semantics: `remove` deletes the +/// precise `(prop_id, value, key)` triple. The SUPERSET behavior lives +/// entirely in string-hash collisions and Bool/Int aliasing (identical +/// to `SegmentPropertyIndexes`) — callers keep a residual Filter +/// downstream regardless, so an index hit can over-select, never +/// under-select. +/// +/// Owned by `MemGraph` (not `NamedGraph`) — same lifetime and same +/// single-writer, no-lock ownership as `nodes`/`node_order` (the shard +/// thread exclusively owns `MemGraph`, so this index needs zero +/// synchronization: no `RwLock`, no atomics). +/// +/// Forward-compat note: any FUTURE property-mutation call site (e.g. a +/// Cypher `REMOVE n.prop` clause, which does not exist yet) MUST route +/// through `MemGraph::set_node_property`/`remove_node_property` rather than +/// poking `MutableNode.properties` directly, or this index goes stale. +#[derive(Debug, Default)] +pub struct MutablePropertyIndex { + /// prop_id -> sorted numeric value -> node keys with that value. + /// `SmallVec<[NodeKey; 2]>` because point-lookup properties (ids) are + /// near-unique in practice; low-cardinality properties spill to heap + /// transparently — no correctness difference, just an allocation, same + /// as today's per-node `SmallVec<4>` properties. + numeric: FxHashMap, SmallVec<[NodeKey; 2]>>>, + /// prop_id -> xxh64(bytes) -> node keys (String/Bytes equality). + strings: FxHashMap>>, +} + +impl MutablePropertyIndex { + /// Insert `(prop_id, value) -> key` into the index. + pub fn insert(&mut self, pid: u16, value: &PropertyValue, key: NodeKey) { + match normalize_numeric(value) { + Some(v) => self + .numeric + .entry(pid) + .or_default() + .entry(OrderedFloat(v)) + .or_default() + .push(key), + None => { + if let PropertyValue::String(s) | PropertyValue::Bytes(s) = value { + self.strings + .entry(pid) + .or_default() + .entry(xxhash_rust::xxh64::xxh64(s, 0)) + .or_default() + .push(key); + } + } + } + } + + /// Remove the exact `(prop_id, value, key)` triple from the index. + /// Cleans up now-empty buckets/prop entries so the index never leaks + /// stale, permanently-empty containers across a long server lifetime. + pub fn remove(&mut self, pid: u16, value: &PropertyValue, key: NodeKey) { + match normalize_numeric(value) { + Some(v) => { + if let Some(tree) = self.numeric.get_mut(&pid) { + let ordered = OrderedFloat(v); + let mut drop_value = false; + if let Some(bucket) = tree.get_mut(&ordered) { + bucket.retain(|k| *k != key); + drop_value = bucket.is_empty(); + } + if drop_value { + tree.remove(&ordered); + } + if tree.is_empty() { + self.numeric.remove(&pid); + } + } + } + None => { + if let PropertyValue::String(s) | PropertyValue::Bytes(s) = value { + if let Some(buckets) = self.strings.get_mut(&pid) { + let hash = xxhash_rust::xxh64::xxh64(s, 0); + let mut drop_hash = false; + if let Some(bucket) = buckets.get_mut(&hash) { + bucket.retain(|k| *k != key); + drop_hash = bucket.is_empty(); + } + if drop_hash { + buckets.remove(&hash); + } + if buckets.is_empty() { + self.strings.remove(&pid); + } + } + } + } + } + } + + /// Index every entry in a node's property map (creation / undelete). + pub fn index_node(&mut self, key: NodeKey, props: &PropertyMap) { + for (pid, val) in props { + self.insert(*pid, val, key); + } + } + + /// Unindex every entry in a node's property map (soft-delete / + /// replace-in-place overwrite). `freeze()` uses `clear()` instead — see + /// its doc comment. + pub fn unindex_node(&mut self, key: NodeKey, props: &PropertyMap) { + for (pid, val) in props { + self.remove(*pid, val, key); + } + } + + /// Zero-alloc equality probe. Returns a borrowed slice (`&[]` when + /// absent) — no candidate materialization until the caller chooses to + /// (`.to_vec()`). + pub fn keys_eq(&self, pid: u16, value: &PropertyValue) -> &[NodeKey] { + match normalize_numeric(value) { + Some(v) => self + .numeric + .get(&pid) + .and_then(|tree| tree.get(&OrderedFloat(v))) + .map(SmallVec::as_slice) + .unwrap_or(&[]), + None => match value { + PropertyValue::String(s) | PropertyValue::Bytes(s) => self + .strings + .get(&pid) + .and_then(|m| m.get(&xxhash_rust::xxh64::xxh64(s, 0))) + .map(SmallVec::as_slice) + .unwrap_or(&[]), + PropertyValue::Int(_) | PropertyValue::Float(_) | PropertyValue::Bool(_) => &[], + }, + } + } + + /// Range probe: `[min, max]` inclusive over the numeric B-tree for + /// `prop_id`. Allocates (BTree range union) — same cost class as + /// `PropertyIndex::range_query`; range queries are not the point-lookup + /// hot path this index primarily targets. + pub fn keys_range(&self, pid: u16, min: f64, max: f64) -> Vec { + let Some(tree) = self.numeric.get(&pid) else { + return Vec::new(); + }; + let lo = OrderedFloat(min); + let hi = OrderedFloat(max); + tree.range(lo..=hi) + .flat_map(|(_, keys)| keys.iter().copied()) + .collect() + } + + /// Drop every indexed entry. `MemGraph::freeze()` drains ALL of + /// `self.nodes` unconditionally (dead or alive), so the entire index is + /// dead the instant freeze starts — `O(#buckets)`, not + /// `O(#nodes * #props)`. + pub fn clear(&mut self) { + self.numeric.clear(); + self.strings.clear(); + } + + /// True when no property is indexed at all. + pub fn is_empty(&self) -> bool { + self.numeric.is_empty() && self.strings.is_empty() + } + + /// Approximate resident bytes: feeds `MemGraph::resident_bytes()`'s + /// elastic-memory-budget accounting (same "monotonic signal, not exact + /// byte accounting" precedent as `PropertyIndex::resident_bytes`). + pub fn resident_bytes(&self) -> usize { + let numeric: usize = self + .numeric + .values() + .flat_map(|tree| tree.values()) + .map(|bucket| { + std::mem::size_of::>() + + bucket.len() * std::mem::size_of::() + }) + .sum(); + let strings: usize = self + .strings + .values() + .flat_map(|inner| inner.values()) + .map(|bucket| { + std::mem::size_of::() + bucket.len() * std::mem::size_of::() + }) + .sum(); + numeric + strings + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -467,6 +700,7 @@ impl SegmentPropertyIndexes { mod tests { use super::*; use crate::graph::types::{EdgeMeta, NodeMeta}; + use bytes::Bytes; // --- LabelIndex tests --- @@ -703,6 +937,14 @@ mod tests { assert_eq!(result.len(), 2); assert!(result.contains(0)); assert!(result.contains(1)); + + // lte(30) -> rows 0, 1, 2 (inclusive upper bound) + let result = idx.lte(30.0); + assert_eq!(result.len(), 3); + assert!(result.contains(2)); + + // lte below the minimum -> empty + assert!(idx.lte(9.0).is_empty()); } #[test] @@ -760,4 +1002,121 @@ mod tests { assert!(result.contains(1)); assert!(!result.contains(2)); } + + // --- MutablePropertyIndex tests --- + + fn make_keys(n: usize) -> Vec { + use slotmap::SlotMap; + let mut sm: SlotMap = SlotMap::with_key(); + (0..n).map(|_| sm.insert(())).collect() + } + + #[test] + fn test_mutable_index_insert_eq_lookup() { + let keys = make_keys(3); + let mut idx = MutablePropertyIndex::default(); + idx.insert(0, &PropertyValue::Int(1), keys[0]); + idx.insert(0, &PropertyValue::Int(2), keys[1]); + idx.insert(0, &PropertyValue::Int(3), keys[2]); + + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(1)), &[keys[0]]); + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(2)), &[keys[1]]); + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(3)), &[keys[2]]); + assert!(idx.keys_eq(0, &PropertyValue::Int(4)).is_empty()); + } + + #[test] + fn test_mutable_index_string_property_eq() { + let keys = make_keys(2); + let mut idx = MutablePropertyIndex::default(); + idx.insert( + 1, + &PropertyValue::String(Bytes::from_static(b"alice")), + keys[0], + ); + idx.insert( + 1, + &PropertyValue::String(Bytes::from_static(b"bob")), + keys[1], + ); + + assert_eq!( + idx.keys_eq(1, &PropertyValue::String(Bytes::from_static(b"alice"))), + &[keys[0]] + ); + assert_eq!( + idx.keys_eq(1, &PropertyValue::String(Bytes::from_static(b"bob"))), + &[keys[1]] + ); + assert!( + idx.keys_eq(1, &PropertyValue::String(Bytes::from_static(b"carol"))) + .is_empty() + ); + } + + #[test] + fn test_mutable_index_update_moves_bucket() { + let keys = make_keys(1); + let k = keys[0]; + let mut idx = MutablePropertyIndex::default(); + idx.insert(0, &PropertyValue::Int(10), k); + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(10)), &[k]); + + // Simulate an update: remove old value, insert new value. + idx.remove(0, &PropertyValue::Int(10), k); + idx.insert(0, &PropertyValue::Int(20), k); + + assert!(idx.keys_eq(0, &PropertyValue::Int(10)).is_empty()); + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(20)), &[k]); + } + + #[test] + fn test_mutable_index_remove_is_exact() { + let keys = make_keys(1); + let k = keys[0]; + let mut idx = MutablePropertyIndex::default(); + idx.insert(0, &PropertyValue::Int(42), k); + idx.remove(0, &PropertyValue::Int(42), k); + assert!(idx.keys_eq(0, &PropertyValue::Int(42)).is_empty()); + // Bucket cleanup: index should be fully empty, not just the value gone. + assert!(idx.is_empty()); + } + + #[test] + fn test_mutable_index_range_query() { + let keys = make_keys(5); + let mut idx = MutablePropertyIndex::default(); + for (i, &k) in keys.iter().enumerate() { + idx.insert(0, &PropertyValue::Int((i as i64 + 1) * 10), k); + } + // Values: 10, 20, 30, 40, 50. Range [20, 40] -> keys[1..=3]. + let mut result = idx.keys_range(0, 20.0, 40.0); + result.sort_by_key(|k| format!("{k:?}")); + let mut expected = vec![keys[1], keys[2], keys[3]]; + expected.sort_by_key(|k| format!("{k:?}")); + assert_eq!(result, expected); + } + + #[test] + fn test_mutable_index_bool_int_aliasing_is_superset() { + let keys = make_keys(1); + let k = keys[0]; + let mut idx = MutablePropertyIndex::default(); + idx.insert(0, &PropertyValue::Bool(true), k); + // Bool(true) normalizes to 1.0, same bucket as Int(1) -- documented + // superset behavior (residual Filter downstream disambiguates). + assert_eq!(idx.keys_eq(0, &PropertyValue::Int(1)), &[k]); + assert_eq!(idx.keys_eq(0, &PropertyValue::Bool(true)), &[k]); + } + + #[test] + fn test_mutable_index_resident_bytes_grows_with_inserts() { + let keys = make_keys(10); + let mut idx = MutablePropertyIndex::default(); + let before = idx.resident_bytes(); + for (i, &k) in keys.iter().enumerate() { + idx.insert(0, &PropertyValue::Int(i as i64), k); + } + assert!(idx.resident_bytes() > before); + } } diff --git a/src/graph/manifest.rs b/src/graph/manifest.rs index f60330803..43786260c 100644 --- a/src/graph/manifest.rs +++ b/src/graph/manifest.rs @@ -40,6 +40,14 @@ pub struct GraphManifest { pub segments: Vec, /// Unix timestamp (seconds) when this manifest was written. pub created_at: u64, + /// Node-id allocation cursor (`MemGraph::id_cursors`) at save time. + /// `0` (absent in pre-cursor manifests) means unknown — recovery then + /// derives the floor from frozen external_ids + WAL replay alone. + #[serde(default)] + pub next_node_id: u64, + /// Edge-id allocation cursor at save time (same contract). + #[serde(default)] + pub next_edge_id: u64, } impl GraphManifest { @@ -47,7 +55,13 @@ impl GraphManifest { /// /// `base_dir` is the relative directory prefix for segment file paths /// (e.g. `"graph_social"`). Segment files are named `seg_{lsn}.csr`. - pub fn from_segments(graph_name: &str, segments: &[Arc], base_dir: &str) -> Self { + /// `id_cursors` is the write buffer's `(next_node_id, next_edge_id)`. + pub fn from_segments( + graph_name: &str, + segments: &[Arc], + base_dir: &str, + id_cursors: (u64, u64), + ) -> Self { let entries: Vec = segments .iter() .map(|seg| SegmentManifestEntry { @@ -69,6 +83,8 @@ impl GraphManifest { graph_name: graph_name.to_owned(), segments: entries, created_at, + next_node_id: id_cursors.0, + next_edge_id: id_cursors.1, } } @@ -130,6 +146,8 @@ mod tests { }, ], created_at: 1700000000, + next_node_id: (7 << 32) | 42, + next_edge_id: (3 << 32) | 9, }; let path = dir.path().join("manifest.json"); @@ -152,9 +170,12 @@ mod tests { let csr = CsrSegment::from_frozen(frozen, 42).expect("ok"); let segments: Vec> = vec![Arc::new(CsrStorage::from(csr))]; - let manifest = GraphManifest::from_segments("test_graph", &segments, "graph_test"); + let manifest = + GraphManifest::from_segments("test_graph", &segments, "graph_test", mg.id_cursors()); assert_eq!(manifest.graph_name, "test_graph"); + assert_eq!(manifest.next_node_id, mg.id_cursors().0); + assert_eq!(manifest.next_edge_id, mg.id_cursors().1); assert_eq!(manifest.segments.len(), 1); assert_eq!(manifest.segments[0].segment_id, 42); assert_eq!(manifest.segments[0].node_count, 2); @@ -185,6 +206,8 @@ mod tests { graph_name: "empty".to_owned(), segments: vec![], created_at: 0, + next_node_id: 0, + next_edge_id: 0, }; let path = dir.path().join("manifest.json"); manifest.save(&path).expect("save ok"); diff --git a/src/graph/memgraph.rs b/src/graph/memgraph.rs index f9884821f..c3931b265 100644 --- a/src/graph/memgraph.rs +++ b/src/graph/memgraph.rs @@ -1,12 +1,42 @@ -//! MemGraph -- mutable adjacency-list write buffer backed by SlotMap. +//! MemGraph -- mutable adjacency-list write buffer with stable external ids. //! //! Absorbs graph writes at O(1) amortized cost per insert. Freezes into a //! `FrozenMemGraph` when the edge threshold is reached, enabling CSR conversion. - -use slotmap::{Key, SlotMap}; +//! +//! Storage is `FxHashMap` keyed by monotonically allocated ids (in slotmap +//! ffi encoding) rather than a `SlotMap`: WAL replay must re-materialize +//! nodes/edges under the EXACT ids that were logged, and slot maps cannot +//! insert at a chosen (index, generation) pair. Insertion order is kept in +//! side vectors so iteration and freeze remain deterministic. + +use crate::graph::fasthash::FxHashMap; +use crate::graph::index::MutablePropertyIndex; use smallvec::SmallVec; -use crate::graph::types::{Direction, EdgeKey, MutableEdge, MutableNode, NodeKey, PropertyMap}; +use crate::graph::types::{ + Direction, EdgeKey, MutableEdge, MutableNode, NodeKey, PropertyMap, PropertyValue, +}; + +/// First id handed out by a fresh MemGraph: index 1, version 1. +/// +/// Ids live in slotmap's ffi encoding (`version << 32 | index`) so they +/// round-trip through `KeyData::from_ffi`/`as_ffi` unchanged. slotmap 1.1 +/// forces the version odd on `from_ffi` (`(value >> 32) | 1`), so allocation +/// stays in odd-version space; index 0 is skipped to avoid the encoding's +/// degenerate all-zero key. +const FIRST_ID: u64 = (1 << 32) | 1; + +/// Next id after `id` in odd-version ffi space. Rolls the 32-bit index over +/// into `version + 2` (stays odd) before it can reach `u32::MAX` (slotmap's +/// null-key index). +#[inline] +const fn next_id(id: u64) -> u64 { + if id as u32 >= u32::MAX - 1 { + ((id >> 32) + 2) << 32 | 1 + } else { + id + 1 + } +} /// Errors returned by MemGraph operations. #[derive(Debug, PartialEq, Eq)] @@ -26,11 +56,22 @@ pub struct FrozenMemGraph { pub edges: Vec<(EdgeKey, MutableEdge)>, } -/// Mutable graph segment backed by generational SlotMap indices. +/// Mutable graph segment keyed by stable, monotonically allocated ids. #[derive(Debug)] pub struct MemGraph { - nodes: SlotMap, - edges: SlotMap, + nodes: FxHashMap, + edges: FxHashMap, + /// Insertion order of live-at-insert node keys (deterministic iteration + /// and freeze). May contain keys later removed from `nodes`. + node_order: Vec, + /// Insertion order of edge keys (same contract as `node_order`). + edge_order: Vec, + /// Next node id to hand out (slotmap ffi encoding, odd version). + /// Monotonic across freeze/thaw — never reset, so post-freeze inserts can + /// never alias the external_ids of frozen CSR rows. + next_node_id: u64, + /// Next edge id to hand out (same encoding and monotonicity contract). + next_edge_id: u64, /// Adjacency (outgoing / incoming) for cross-tier "delta" edges whose /// endpoint was frozen into a CSR segment. A frozen endpoint has no /// `MutableNode` to carry inline adjacency, so its edge keys live here, @@ -44,154 +85,166 @@ pub struct MemGraph { /// Edge count threshold that triggers freeze. edge_threshold: usize, frozen: bool, - /// Index-space watermark added to the raw SlotMap index of every - /// NodeKey this MemGraph hands out or accepts. See `with_id_offset` for - /// the full soundness argument. Zero (the default via `new`) makes the - /// translation the identity function -- the overwhelmingly common case - /// for graphs with no persisted (pre-restart) history. - id_offset: u32, + /// Incrementally-maintained property index over the mutable tier (Task + /// #31 — see `tmp/DESIGN-MUTABLE-PROP-INDEX.md`). Owned here (not on + /// `NamedGraph`) for the same single-writer, no-lock lifetime as + /// `nodes`/`node_order`. Maintained by `insert_node_at`, `remove_node`, + /// `set_node_property`, `remove_node_property`, `undelete_node`, and + /// wholesale-cleared at `freeze()`. + prop_index: MutablePropertyIndex, } impl MemGraph { /// Create an empty MemGraph with the given freeze threshold. - pub fn new(edge_threshold: usize) -> Self { - Self::with_id_offset(edge_threshold, 0) - } - - /// Create an empty MemGraph whose NodeKeys are all minted `id_offset` - /// above the raw SlotMap index. - /// - /// # Why this exists (soundness argument -- graph NodeKey aliasing, P0) - /// - /// `slotmap::SlotMap` key allocation is fully deterministic and a fresh - /// map's index counter always starts at 0 - /// (`slotmap::basic::SlotMap::insert`, free_head/grow-path). After a - /// restart, WAL replay works against a fresh `MemGraph` - /// (`replay.rs::take_memgraph`), while CSR segments loaded from disk - /// carry `NodeMeta::external_id` values that are the raw - /// `slotmap::KeyData::as_ffi()` bits minted by the PRE-CRASH process's - /// SlotMap -- which also started at index 0. Without an offset, the - /// first node the fresh MemGraph mints gets `(idx=0, version=1)`, - /// bit-for-bit IDENTICAL to the pre-crash process's first-ever node key - /// if that node is still resident in a loaded CSR segment (the common - /// case whenever an AOF fold/WAL checkpoint truncates pre-freeze - /// history so replay only sees post-freeze commands). Every merged read - /// path (`MergedNodeView`) checks the mutable tier first, so the - /// aliasing new node would permanently and silently shadow the real - /// frozen node. - /// - /// ## Fix - /// - /// Shift the INDEX component (low 32 bits of `KeyData::as_ffi()`, see - /// `slotmap::KeyData::{as_ffi, from_ffi}` -- version occupies the high - /// 32 bits) of every key this MemGraph hands out by `id_offset`. The - /// caller (recovery.rs) chooses `id_offset` to be `> ` the largest index - /// component among ALL `external_id`s in the CSR segments it just - /// loaded for this graph: - /// - Outgoing (`add_node` return value, `iter_nodes` keys): raw SlotMap - /// index + `id_offset` (see `to_public_key`). - /// - Incoming (any NodeKey parameter): raw index = public index - - /// `id_offset`; underflow (public index < `id_offset`) means the key - /// was never minted by THIS MemGraph -- it is a CSR-only (or foreign) - /// key -- and is treated as "not resident", exactly the existing - /// ghost / `NodeNotFound` semantics already used for non-resident - /// endpoints (see `to_internal_key`). - /// - /// This costs two `u64` shifts per NodeKey touched (zero when - /// `id_offset == 0`) and precisely **zero** extra permanent memory: - /// the alternative of pre-consuming `id_offset` SlotMap slots via a - /// dummy-insert/remove cycle would permanently pin `id_offset` - /// live-or-vacant slot entries in the SlotMap's backing `Vec` (slotmap - /// never shrinks its storage) -- exactly the O(watermark) leak this - /// design avoids. It is also the only sound option: a dummy-cycle - /// approach only bumps a slot's *generation*, and persisted external_ids - /// may already carry an arbitrarily-bumped generation from pre-crash - /// slot churn, so a fixed number of dummy cycles cannot be proven to - /// out-run every possible persisted generation for a given index. - /// - /// ## Overflow /// - /// If `idx + id_offset` would exceed `u32::MAX`, the public key - /// saturates at `u32::MAX` instead of wrapping (a wrapped index could - /// re-enter `[0, id_offset)` and alias a persisted `external_id` again). - /// This is an astronomical corner case (>4 billion prior nodes on one - /// graph) and degrades to "new inserts stop being independently - /// addressable" rather than corrupting existing data. - pub fn with_id_offset(edge_threshold: usize, id_offset: u32) -> Self { + /// Restart NodeKey-aliasing soundness (P0): node/edge ids are handed out + /// from explicit monotonic cursors (`next_node_id`/`next_edge_id`) that + /// recovery restores from the manifest AND floors past every frozen + /// `external_id` (`restore_id_cursors` + `ensure_node_id_floor`), while + /// WAL replay re-materializes nodes under their ORIGINAL logged ids + /// (`add_node_with_id`). A fresh post-restart insert therefore can never + /// mint a key that numerically aliases a persisted frozen row — the bug + /// the earlier SlotMap representation had (deterministic index counter + /// restarting at 0). See `tests/graph_restart_id_aliasing.rs`. + pub fn new(edge_threshold: usize) -> Self { Self { - nodes: SlotMap::with_key(), - edges: SlotMap::with_key(), + nodes: FxHashMap::default(), + edges: FxHashMap::default(), + node_order: Vec::new(), + edge_order: Vec::new(), + next_node_id: FIRST_ID, + next_edge_id: FIRST_ID, ghost_out: std::collections::HashMap::new(), ghost_in: std::collections::HashMap::new(), live_node_count: 0, live_edge_count: 0, edge_threshold, frozen: false, - id_offset, + prop_index: MutablePropertyIndex::default(), } } - /// Translate a raw internal SlotMap `NodeKey` to the PUBLIC key handed - /// to callers (index + `id_offset`, version unchanged). Identity when - /// `id_offset == 0`. See `with_id_offset` for the soundness argument. - #[inline] - fn to_public_key(id_offset: u32, key: NodeKey) -> NodeKey { - if id_offset == 0 { - return key; - } - let ffi = key.data().as_ffi(); - let idx = ffi as u32; - let version = ffi >> 32; - // Saturate rather than wrap: a wrapped index could re-enter - // [0, id_offset) and alias a persisted external_id again. - let public_idx = idx.saturating_add(id_offset); - NodeKey::from(slotmap::KeyData::from_ffi( - (version << 32) | u64::from(public_idx), - )) - } - - /// Translate a PUBLIC `NodeKey` (offset applied) to the raw internal - /// SlotMap key used to index `self.nodes`. Returns `None` if the public - /// index is below `id_offset` -- such a key was never minted by this - /// MemGraph (it belongs to a CSR segment or a foreign graph) and must be - /// treated as non-resident, matching the existing ghost / NodeNotFound - /// semantics for non-resident endpoints. Identity when `id_offset == 0`. - #[inline] - fn to_internal_key(id_offset: u32, key: NodeKey) -> Option { - if id_offset == 0 { - return Some(key); - } - let ffi = key.data().as_ffi(); - let idx = ffi as u32; - let version = ffi >> 32; - let internal_idx = idx.checked_sub(id_offset)?; - Some(NodeKey::from(slotmap::KeyData::from_ffi( - (version << 32) | u64::from(internal_idx), - ))) + /// Insert a new node under a freshly allocated stable id. + pub fn add_node( + &mut self, + labels: SmallVec<[u16; 4]>, + properties: PropertyMap, + embedding: Option>, + lsn: u64, + ) -> NodeKey { + let id = self.next_node_id; + self.next_node_id = next_id(id); + self.insert_node_at( + NodeKey::from(slotmap::KeyData::from_ffi(id)), + labels, + properties, + embedding, + lsn, + ) } - /// Insert a new node. Returns the generational (public) key. - pub fn add_node( + /// Insert a new node under a CALLER-CHOSEN id (WAL replay: the id that + /// was logged at original execution). Bumps the allocation floor past + /// `node_id` so later `add_node` calls can never alias it. Replaces any + /// existing entry under the same id (replay is authoritative). + pub fn add_node_with_id( &mut self, + node_id: u64, labels: SmallVec<[u16; 4]>, properties: PropertyMap, embedding: Option>, lsn: u64, ) -> NodeKey { - let key = self.nodes.insert(MutableNode { + self.ensure_node_id_floor(node_id); + self.insert_node_at( + NodeKey::from(slotmap::KeyData::from_ffi(node_id)), labels, - outgoing: SmallVec::new(), - incoming: SmallVec::new(), properties, embedding, - created_lsn: lsn, - deleted_lsn: u64::MAX, - txn_id: 0, - valid_from: 0, - valid_to: i64::MAX, - }); - self.live_node_count += 1; - Self::to_public_key(self.id_offset, key) + lsn, + ) + } + + /// Raise the node-id allocation floor above `node_id` (no-op if already + /// above). Used when frozen CSR external_ids are re-seeded at recovery. + pub fn ensure_node_id_floor(&mut self, node_id: u64) { + if node_id >= self.next_node_id { + self.next_node_id = next_id(node_id); + } + } + + /// Raise the edge-id allocation floor above `edge_id`. + pub fn ensure_edge_id_floor(&mut self, edge_id: u64) { + if edge_id >= self.next_edge_id { + self.next_edge_id = next_id(edge_id); + } + } + + /// Current allocation cursors `(next_node_id, next_edge_id)` — persisted + /// in the graph manifest so a restart resumes allocation past every id + /// ever handed out (WAL-truncation-safe). + pub fn id_cursors(&self) -> (u64, u64) { + (self.next_node_id, self.next_edge_id) + } + + /// Restore persisted allocation cursors. Values are `next_*` cursors + /// (NOT handed-out ids — see `ensure_*_id_floor` for those). Only ever + /// raises; `0` (pre-cursor manifest) is a no-op. + pub fn restore_id_cursors(&mut self, next_node_id: u64, next_edge_id: u64) { + if next_node_id > self.next_node_id { + self.next_node_id = next_node_id; + } + if next_edge_id > self.next_edge_id { + self.next_edge_id = next_edge_id; + } + } + + fn insert_node_at( + &mut self, + key: NodeKey, + labels: SmallVec<[u16; 4]>, + properties: PropertyMap, + embedding: Option>, + lsn: u64, + ) -> NodeKey { + // Index the NEW properties before the move below so a fresh insert + // never has a transient window with zero index presence. + self.prop_index.index_node(key, &properties); + let prev = self.nodes.insert( + key, + MutableNode { + labels, + outgoing: SmallVec::new(), + incoming: SmallVec::new(), + properties, + embedding, + created_lsn: lsn, + deleted_lsn: u64::MAX, + txn_id: 0, + valid_from: 0, + valid_to: i64::MAX, + }, + ); + match prev { + Some(old) => { + // Replaced-in-place (replay overwrite): order entry already + // present; only fix the live count if the old entry was dead. + if old.deleted_lsn != u64::MAX { + self.live_node_count += 1; + } + // Undo the double-index from a replace: the OLD value's + // bucket entry must not dangle when the same id is + // replayed twice under different property values. Harmless + // no-op if `old` was already dead (its properties were + // already unindexed by `remove_node`'s soft-delete). + self.prop_index.unindex_node(key, &old.properties); + } + None => { + self.node_order.push(key); + self.live_node_count += 1; + } + } + key } /// Insert a new edge between `src` and `dst`. Validates both exist and are alive. @@ -210,33 +263,21 @@ impl MemGraph { if src == dst { return Err(GraphError::SelfLoop); } - // Translate PUBLIC keys to raw internal SlotMap keys. Translation - // failure (offset underflow) means the key was never minted by this - // MemGraph -- treat exactly like "not resident" (`add_edge` does not + // Validate both nodes exist and are alive (`add_edge` does not // support cross-tier endpoints; use `add_edge_across_tiers`). - let Some(src_i) = Self::to_internal_key(self.id_offset, src) else { - return Err(GraphError::NodeNotFound); - }; - let Some(dst_i) = Self::to_internal_key(self.id_offset, dst) else { - return Err(GraphError::NodeNotFound); - }; - // Validate both nodes exist and are alive. let src_alive = self .nodes - .get(src_i) + .get(&src) .map_or(false, |n| n.deleted_lsn == u64::MAX); let dst_alive = self .nodes - .get(dst_i) + .get(&dst) .map_or(false, |n| n.deleted_lsn == u64::MAX); if !src_alive || !dst_alive { return Err(GraphError::NodeNotFound); } - let ek = self.edges.insert(MutableEdge { - // Stored as PUBLIC keys: `MutableEdge.src/dst` are the identity - // callers (freeze(), neighbors()) compare against, matching - // `add_node`'s return value and CSR `external_id`s. + let ek = self.insert_edge_fresh(MutableEdge { src, dst, edge_type, @@ -254,16 +295,27 @@ impl MemGraph { // Push edge key into src.outgoing and dst.incoming. // Both are validated alive above, so get_mut is safe. - if let Some(src_node) = self.nodes.get_mut(src_i) { + if let Some(src_node) = self.nodes.get_mut(&src) { src_node.outgoing.push(ek); } - if let Some(dst_node) = self.nodes.get_mut(dst_i) { + if let Some(dst_node) = self.nodes.get_mut(&dst) { dst_node.incoming.push(ek); } self.live_edge_count += 1; Ok(ek) } + /// Insert an edge under a freshly allocated stable id and record its + /// insertion order. Does NOT touch adjacency or live counts. + fn insert_edge_fresh(&mut self, edge: MutableEdge) -> EdgeKey { + let id = self.next_edge_id; + self.next_edge_id = next_id(id); + let ek = EdgeKey::from(slotmap::KeyData::from_ffi(id)); + self.edges.insert(ek, edge); + self.edge_order.push(ek); + ek + } + /// Insert an edge whose endpoints may live in the frozen CSR tier /// (a "delta" edge). Resident endpoints are validated alive and get /// inline adjacency; non-resident endpoints get ghost adjacency. @@ -279,6 +331,38 @@ impl MemGraph { weight: f64, properties: Option, lsn: u64, + ) -> Result { + self.add_edge_across_tiers_at(None, src, dst, edge_type, weight, properties, lsn) + } + + /// `add_edge_across_tiers` under a CALLER-CHOSEN edge id (WAL replay). + /// Bumps the edge-id allocation floor past `edge_id`. + #[allow(clippy::too_many_arguments)] + pub fn add_edge_across_tiers_with_id( + &mut self, + edge_id: u64, + src: NodeKey, + dst: NodeKey, + edge_type: u16, + weight: f64, + properties: Option, + lsn: u64, + ) -> Result { + self.ensure_edge_id_floor(edge_id); + let ek = EdgeKey::from(slotmap::KeyData::from_ffi(edge_id)); + self.add_edge_across_tiers_at(Some(ek), src, dst, edge_type, weight, properties, lsn) + } + + #[allow(clippy::too_many_arguments)] + fn add_edge_across_tiers_at( + &mut self, + chosen: Option, + src: NodeKey, + dst: NodeKey, + edge_type: u16, + weight: f64, + properties: Option, + lsn: u64, ) -> Result { if self.frozen { return Err(GraphError::AlreadyFrozen); @@ -286,23 +370,17 @@ impl MemGraph { if src == dst { return Err(GraphError::SelfLoop); } - // Translate PUBLIC keys to internal SlotMap keys where possible. - // `None` (translation underflow, or a key genuinely absent from this - // MemGraph) means non-resident -- caller-verified via the CSR tier. - let src_i = Self::to_internal_key(self.id_offset, src); - let dst_i = Self::to_internal_key(self.id_offset, dst); - // Resident endpoints must be alive; non-resident are caller-verified. - for key_i in [src_i, dst_i].into_iter().flatten() { - if let Some(n) = self.nodes.get(key_i) { + // Resident endpoints must be alive; non-resident are caller-verified + // via the CSR tier (delta edge). + for key in [src, dst] { + if let Some(n) = self.nodes.get(&key) { if n.deleted_lsn != u64::MAX { return Err(GraphError::NodeNotFound); } } } - let ek = self.edges.insert(MutableEdge { - // PUBLIC keys: ghost_out/ghost_in are keyed by public identity - // too (a non-resident endpoint has no internal key at all). + let edge = MutableEdge { src, dst, edge_type, @@ -314,13 +392,33 @@ impl MemGraph { valid_from: 0, valid_to: i64::MAX, created_ms: crate::storage::entry::current_time_ms(), - }); + }; + let ek = match chosen { + Some(ek) => { + match self.edges.insert(ek, edge) { + None => self.edge_order.push(ek), + Some(old) => { + // Replay overwrite of an identical logged edge id: + // adjacency already points at `ek`; keep counts + // consistent (the +1 below re-adds a live entry). + if old.deleted_lsn == u64::MAX { + self.live_edge_count = self.live_edge_count.saturating_sub(1); + } + // Skip the adjacency pushes — already linked. + self.live_edge_count += 1; + return Ok(ek); + } + } + ek + } + None => self.insert_edge_fresh(edge), + }; - match src_i.and_then(|k| self.nodes.get_mut(k)) { + match self.nodes.get_mut(&src) { Some(src_node) => src_node.outgoing.push(ek), None => self.ghost_out.entry(src).or_default().push(ek), } - match dst_i.and_then(|k| self.nodes.get_mut(k)) { + match self.nodes.get_mut(&dst) { Some(dst_node) => dst_node.incoming.push(ek), None => self.ghost_in.entry(dst).or_default().push(ek), } @@ -330,10 +428,7 @@ impl MemGraph { /// Soft-delete a node and all its incident edges. pub fn remove_node(&mut self, key: NodeKey, lsn: u64) -> bool { - let Some(internal) = Self::to_internal_key(self.id_offset, key) else { - return false; - }; - let Some(node) = self.nodes.get_mut(internal) else { + let Some(node) = self.nodes.get_mut(&key) else { return false; }; if node.deleted_lsn != u64::MAX { @@ -350,9 +445,16 @@ impl MemGraph { .copied() .collect(); + // Soft-delete removes the node from live property-index candidacy. + // Disjoint-field borrow: `node` borrows `self.nodes` via `get_mut`, + // `self.prop_index` is a distinct field — legal without + // restructuring since both accesses go through explicit field + // projections, not an opaque `self.foo()` call. + self.prop_index.unindex_node(key, &node.properties); + // Soft-delete all incident edges. for ek in edge_keys { - if let Some(edge) = self.edges.get_mut(ek) { + if let Some(edge) = self.edges.get_mut(&ek) { if edge.deleted_lsn == u64::MAX { edge.deleted_lsn = lsn; self.live_edge_count = self.live_edge_count.saturating_sub(1); @@ -364,7 +466,7 @@ impl MemGraph { /// Soft-delete a single edge. pub fn remove_edge(&mut self, key: EdgeKey, lsn: u64) -> bool { - let Some(edge) = self.edges.get_mut(key) else { + let Some(edge) = self.edges.get_mut(&key) else { return false; }; if edge.deleted_lsn != u64::MAX { @@ -386,24 +488,107 @@ impl MemGraph { /// O(1) node lookup by key. pub fn get_node(&self, key: NodeKey) -> Option<&MutableNode> { - let internal = Self::to_internal_key(self.id_offset, key)?; - self.nodes.get(internal) + self.nodes.get(&key) } /// O(1) mutable node lookup by key. pub fn get_node_mut(&mut self, key: NodeKey) -> Option<&mut MutableNode> { - let internal = Self::to_internal_key(self.id_offset, key)?; - self.nodes.get_mut(internal) + self.nodes.get_mut(&key) } /// O(1) edge lookup by key. pub fn get_edge(&self, key: EdgeKey) -> Option<&MutableEdge> { - self.edges.get(key) + self.edges.get(&key) } /// O(1) mutable edge lookup by key. pub fn get_edge_mut(&mut self, key: EdgeKey) -> Option<&mut MutableEdge> { - self.edges.get_mut(key) + self.edges.get_mut(&key) + } + + /// Set (insert-or-overwrite) a node's property, keeping the mutable-tier + /// property index in sync. Returns the OLD value (`None` if the node was + /// missing OR the property was newly added — same "no prior value" + /// signal callers already treat identically for undo/WAL bookkeeping). + /// + /// Single source of truth for node-property mutation: every call site + /// that used to hand-roll a find-and-replace-or-push on + /// `MutableNode.properties` (SET clause, MERGE ON CREATE/MATCH SET, + /// TXN.ABORT undo, WAL replay) MUST route through this method instead, + /// or the property index silently goes stale. + pub fn set_node_property( + &mut self, + key: NodeKey, + pid: u16, + value: PropertyValue, + ) -> Option { + let node = self.nodes.get_mut(&key)?; + let old = node + .properties + .iter() + .find(|(k, _)| *k == pid) + .map(|(_, v)| v.clone()); + match node.properties.iter_mut().find(|(k, _)| *k == pid) { + Some(entry) => entry.1 = value.clone(), + None => node.properties.push((pid, value.clone())), + } + if let Some(old_v) = &old { + self.prop_index.remove(pid, old_v, key); + } + self.prop_index.insert(pid, &value, key); + old + } + + /// Remove a property entirely, keeping the mutable-tier index in sync. + /// Returns the removed value, if any (TXN.ABORT's "property did not + /// exist before SET" undo branch; also the future landing spot for a + /// Cypher `REMOVE n.prop` clause, which does not exist yet). + pub fn remove_node_property(&mut self, key: NodeKey, pid: u16) -> Option { + let node = self.nodes.get_mut(&key)?; + let removed = node + .properties + .iter() + .position(|(k, _)| *k == pid) + .map(|i| node.properties.remove(i).1); + if let Some(v) = &removed { + self.prop_index.remove(pid, v, key); + } + removed + } + + /// Un-soft-delete a node previously removed at `delete_lsn` (TXN.ABORT's + /// `UndeleteNode` undo), re-indexing its CURRENT properties. Returns + /// `false` (no-op) unless the node is soft-deleted at EXACTLY + /// `delete_lsn` — idempotent re-entry guard, same contract the call site + /// enforced by hand before this method centralized it. + /// + /// This closes a gap a naive `deleted_lsn = u64::MAX` flip would leave: + /// `remove_node` unindexes on soft-delete, so undoing that delete + /// without re-indexing would leave the node live but permanently + /// invisible to `MATCH {prop: val}` index probes. + pub fn undelete_node(&mut self, key: NodeKey, delete_lsn: u64) -> bool { + let Some(node) = self.nodes.get_mut(&key) else { + return false; + }; + if node.deleted_lsn != delete_lsn { + return false; + } + node.deleted_lsn = u64::MAX; + self.live_node_count += 1; + self.prop_index.index_node(key, &node.properties); + true + } + + /// Zero-alloc equality probe into the mutable-tier property index. See + /// `MutablePropertyIndex::keys_eq`. + pub fn prop_index_keys_eq(&self, pid: u16, value: &PropertyValue) -> &[NodeKey] { + self.prop_index.keys_eq(pid, value) + } + + /// Range probe into the mutable-tier property index (allocates). See + /// `MutablePropertyIndex::keys_range`. + pub fn prop_index_keys_range(&self, pid: u16, min: f64, max: f64) -> Vec { + self.prop_index.keys_range(pid, min, max) } /// Returns neighbors of `node` visible at the given `lsn`, filtered by direction. @@ -411,8 +596,7 @@ impl MemGraph { /// Yields `(EdgeKey, NodeKey)` pairs -- the edge and the neighbor node. /// No heap allocation: iterates over borrowed SmallVec adjacency lists. pub fn neighbors(&self, node: NodeKey, direction: Direction, lsn: u64) -> NeighborIter<'_> { - let internal = Self::to_internal_key(self.id_offset, node); - let Some(n) = internal.and_then(|k| self.nodes.get(k)) else { + let Some(n) = self.nodes.get(&node) else { // Non-resident (frozen) node: serve delta-edge adjacency from the // ghost maps so cross-tier edges are traversable from BOTH ends. let ghost_out = match direction { @@ -455,19 +639,31 @@ impl MemGraph { } } - /// Iterate over all live (non-deleted) nodes. Yields `(NodeKey, &MutableNode)` - /// with PUBLIC keys (offset applied) -- matching `add_node`'s return value. + /// Iterate over all live (non-deleted) nodes in insertion order. + /// Yields `(NodeKey, &MutableNode)`. pub fn iter_nodes(&self) -> impl Iterator { - let id_offset = self.id_offset; - self.nodes + self.node_order .iter() + .filter_map(move |k| self.nodes.get(k).map(|n| (*k, n))) .filter(|(_, n)| n.deleted_lsn == u64::MAX) - .map(move |(k, n)| (Self::to_public_key(id_offset, k), n)) } - /// Iterate over all live (non-deleted) edges. Yields `(EdgeKey, &MutableEdge)`. + /// Iterate over all live (non-deleted) edges in insertion order. + /// Yields `(EdgeKey, &MutableEdge)`. pub fn iter_edges(&self) -> impl Iterator { - self.edges.iter().filter(|(_, e)| e.deleted_lsn == u64::MAX) + self.edge_order + .iter() + .filter_map(move |k| self.edges.get(k).map(|e| (*k, e))) + .filter(|(_, e)| e.deleted_lsn == u64::MAX) + } + + /// Iterate over all soft-deleted nodes in insertion order (copy-up + /// tombstone bookkeeping at freeze time). + pub fn iter_dead_nodes(&self) -> impl Iterator { + self.node_order + .iter() + .filter_map(move |k| self.nodes.get(k).map(|n| (*k, n))) + .filter(|(_, n)| n.deleted_lsn != u64::MAX) } /// Number of live (non-deleted) nodes. O(1) via maintained counter. @@ -498,10 +694,7 @@ impl MemGraph { /// were cascade-deleted at `lsn` by `remove_node`. Restores `deleted_lsn` /// to `u64::MAX` and increments `live_edge_count` for each restored edge. pub fn undelete_edges_at_lsn(&mut self, node: NodeKey, lsn: u64) { - let Some(internal) = Self::to_internal_key(self.id_offset, node) else { - return; - }; - let Some(n) = self.nodes.get(internal) else { + let Some(n) = self.nodes.get(&node) else { return; }; let edge_keys: SmallVec<[EdgeKey; 16]> = n @@ -511,7 +704,7 @@ impl MemGraph { .copied() .collect(); for ek in edge_keys { - if let Some(edge) = self.edges.get_mut(ek) { + if let Some(edge) = self.edges.get_mut(&ek) { if edge.deleted_lsn == lsn { edge.deleted_lsn = u64::MAX; self.live_edge_count += 1; @@ -520,12 +713,16 @@ impl MemGraph { } } - /// Resident bytes used by the in-memory adjacency lists (nodes + edges - /// slot maps). Approximation based on slot-map capacity and struct sizes. + /// Resident bytes used by the in-memory adjacency maps (nodes + edges). + /// Approximation based on map capacity and struct sizes. pub fn resident_bytes(&self) -> usize { - let node_bytes = self.nodes.capacity() * std::mem::size_of::(); - let edge_bytes = self.edges.capacity() * std::mem::size_of::(); - node_bytes + edge_bytes + let node_bytes = self.nodes.capacity() + * (std::mem::size_of::() + std::mem::size_of::()) + + self.node_order.capacity() * std::mem::size_of::(); + let edge_bytes = self.edges.capacity() + * (std::mem::size_of::() + std::mem::size_of::()) + + self.edge_order.capacity() * std::mem::size_of::(); + node_bytes + edge_bytes + self.prop_index.resident_bytes() } /// Whether the MemGraph should be frozen (threshold reached). @@ -549,51 +746,57 @@ impl MemGraph { } self.frozen = true; + // The property index is derived entirely from `self.nodes`, which + // this function unconditionally drains in full below (dead or + // alive) — clear it up front rather than per-node: O(#buckets), not + // O(#nodes * #props), and correct because nothing reads the index + // between here and the drain completing (single-threaded, no + // background compaction thread for graphs). + self.prop_index.clear(); + // Partition edges BEFORE draining nodes (residency check needs the - // slot map): live + both endpoints resident → freeze into CSR; + // node map): live + both endpoints resident → freeze into CSR; // live + any frozen-elsewhere endpoint → retain as delta; - // dead → drop. + // dead → drop. Insertion order preserved via edge_order. let mut freeze_keys: Vec = Vec::new(); let mut dead_keys: Vec = Vec::new(); - let id_offset = self.id_offset; - // `e.src`/`e.dst` are PUBLIC keys; translate to internal before - // checking slot-map residency. - let is_resident = |nodes: &SlotMap, key: NodeKey| { - Self::to_internal_key(id_offset, key).is_some_and(|k| nodes.contains_key(k)) - }; - for (ek, e) in self.edges.iter() { + for ek in &self.edge_order { + let Some(e) = self.edges.get(ek) else { + continue; + }; if e.deleted_lsn != u64::MAX { - dead_keys.push(ek); - } else if is_resident(&self.nodes, e.src) && is_resident(&self.nodes, e.dst) { - freeze_keys.push(ek); + dead_keys.push(*ek); + } else if self.nodes.contains_key(&e.src) && self.nodes.contains_key(&e.dst) { + freeze_keys.push(*ek); } } for ek in dead_keys { - self.edges.remove(ek); + self.edges.remove(&ek); } let edges: Vec<(EdgeKey, MutableEdge)> = freeze_keys .into_iter() - .filter_map(|ek| self.edges.remove(ek).map(|e| (ek, e))) + .filter_map(|ek| self.edges.remove(&ek).map(|e| (ek, e))) .collect(); + self.edge_order.retain(|ek| self.edges.contains_key(ek)); - // `drain()` yields raw internal keys -- translate to PUBLIC before - // handing them to CSR conversion (they become `NodeMeta::external_id` - // and must match the identity every other reference to this node - // uses: node_map, ghost adjacency, edge endpoints). - let nodes: Vec<(NodeKey, MutableNode)> = self - .nodes - .drain() - .filter(|(_, n)| n.deleted_lsn == u64::MAX) - .map(|(k, n)| (Self::to_public_key(id_offset, k), n)) - .collect(); + let mut nodes: Vec<(NodeKey, MutableNode)> = Vec::with_capacity(self.nodes.len()); + for nk in self.node_order.drain(..) { + if let Some(n) = self.nodes.remove(&nk) { + if n.deleted_lsn == u64::MAX { + nodes.push((nk, n)); + } + } + } // Rebuild ghost adjacency for the retained delta edges: with the - // node slot map drained, EVERY endpoint is now non-resident. + // node map drained, EVERY endpoint is now non-resident. self.ghost_out.clear(); self.ghost_in.clear(); - for (ek, e) in self.edges.iter() { - self.ghost_out.entry(e.src).or_default().push(ek); - self.ghost_in.entry(e.dst).or_default().push(ek); + for ek in &self.edge_order { + if let Some(e) = self.edges.get(ek) { + self.ghost_out.entry(e.src).or_default().push(*ek); + self.ghost_in.entry(e.dst).or_default().push(*ek); + } } Ok(FrozenMemGraph { nodes, edges }) @@ -601,12 +804,15 @@ impl MemGraph { /// Re-arm a drained MemGraph for writes after a successful freeze. /// - /// Keeps the SAME slot maps (drain bumps each vacated slot's generation, - /// so keys handed out after thaw can never collide with the external_ids - /// of frozen CSR rows). Replacing the MemGraph with a fresh one instead - /// would restart SlotMap allocation at the same (index, generation) pairs - /// and silently alias new nodes onto frozen rows. + /// The monotonic id counters are NOT reset, so keys handed out after + /// thaw can never collide with the external_ids of frozen CSR rows. + /// (Replacing the MemGraph with a fresh one instead would restart + /// allocation at FIRST_ID and silently alias new nodes onto frozen rows.) pub fn thaw(&mut self) { + debug_assert!( + self.prop_index.is_empty(), + "prop_index must be empty after freeze() drained all nodes" + ); self.frozen = false; // Post-freeze contents: zero nodes, retained delta edges (all live — // freeze removed dead ones). @@ -617,7 +823,7 @@ impl MemGraph { /// Zero-allocation neighbor iterator. Borrows from MemGraph's SmallVec adjacency lists. pub struct NeighborIter<'a> { - edges: &'a SlotMap, + edges: &'a FxHashMap, out_iter: core::slice::Iter<'a, EdgeKey>, in_iter: core::slice::Iter<'a, EdgeKey>, lsn: u64, @@ -650,7 +856,7 @@ impl<'a> Iterator for NeighborIter<'a> { }; loop { if let Some(&ek) = self.out_iter.next() { - if let Some(edge) = self.edges.get(ek) { + if let Some(edge) = self.edges.get(&ek) { if is_visible(edge) { return Some((ek, edge.dst)); } @@ -658,7 +864,7 @@ impl<'a> Iterator for NeighborIter<'a> { continue; } if let Some(&ek) = self.in_iter.next() { - if let Some(edge) = self.edges.get(ek) { + if let Some(edge) = self.edges.get(&ek) { if is_visible(edge) { return Some((ek, edge.src)); } @@ -892,4 +1098,128 @@ mod tests { GraphError::NodeNotFound ); } + + // --- MutablePropertyIndex maintenance-hook tests (Task #31) --- + + use crate::graph::types::PropertyValue; + + fn id_props(id: i64) -> PropertyMap { + smallvec![(0u16, PropertyValue::Int(id))] + } + + #[test] + fn test_add_node_populates_prop_index() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(42), None, 1); + assert_eq!(g.prop_index_keys_eq(0, &PropertyValue::Int(42)), &[a]); + assert!(g.prop_index_keys_eq(0, &PropertyValue::Int(43)).is_empty()); + } + + #[test] + fn test_remove_node_clears_prop_index() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(42), None, 1); + g.remove_node(a, 2); + assert!( + g.prop_index_keys_eq(0, &PropertyValue::Int(42)).is_empty(), + "soft-deleted node must not remain an index candidate" + ); + } + + #[test] + fn test_replay_overwrite_unindexes_old_properties() { + // Simulates WAL replay: the SAME node id is (re-)inserted twice + // under different property values. This is the leak edge case the + // `Some(old)` branch in `insert_node_at` exists to close. + let mut g = MemGraph::new(1000); + let a = g.add_node_with_id((1u64 << 32) | 1, smallvec![0], id_props(1), None, 1); + assert_eq!(g.prop_index_keys_eq(0, &PropertyValue::Int(1)), &[a]); + + let a2 = g.add_node_with_id((1u64 << 32) | 1, smallvec![0], id_props(2), None, 2); + assert_eq!(a, a2, "same ffi id must re-materialize the same NodeKey"); + + assert!( + g.prop_index_keys_eq(0, &PropertyValue::Int(1)).is_empty(), + "old value's bucket must not dangle after a replay overwrite" + ); + assert_eq!(g.prop_index_keys_eq(0, &PropertyValue::Int(2)), &[a]); + } + + #[test] + fn test_freeze_clears_prop_index() { + let mut g = MemGraph::new(1000); + g.add_node(smallvec![0], id_props(1), None, 1); + g.add_node(smallvec![0], id_props(2), None, 1); + g.freeze().expect("freeze ok"); + assert!(g.prop_index.is_empty()); + // thaw()'s debug_assert must not fire (stays empty post-thaw). + g.thaw(); + assert!(g.prop_index.is_empty()); + } + + #[test] + fn test_set_node_property_moves_index_entry() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(1), None, 1); + let old = g.set_node_property(a, 0, PropertyValue::Int(2)); + assert_eq!(old, Some(PropertyValue::Int(1))); + assert!(g.prop_index_keys_eq(0, &PropertyValue::Int(1)).is_empty()); + assert_eq!(g.prop_index_keys_eq(0, &PropertyValue::Int(2)), &[a]); + } + + #[test] + fn test_set_node_property_new_key_returns_none() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], empty_props(), None, 1); + let old = g.set_node_property(a, 5, PropertyValue::Int(9)); + assert_eq!(old, None); + assert_eq!(g.prop_index_keys_eq(5, &PropertyValue::Int(9)), &[a]); + } + + #[test] + fn test_remove_node_property_clears_index_entry() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(7), None, 1); + let removed = g.remove_node_property(a, 0); + assert_eq!(removed, Some(PropertyValue::Int(7))); + assert!(g.prop_index_keys_eq(0, &PropertyValue::Int(7)).is_empty()); + assert_eq!(g.get_node(a).expect("node").properties.len(), 0); + } + + #[test] + fn test_undelete_node_reindexes_properties() { + // Targets the TXN.ABORT UndeleteNode gap (design doc risk #6): a + // naive `deleted_lsn = u64::MAX` flip without re-indexing would + // leave the node live but invisible to index probes forever. + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(1), None, 1); + g.remove_node(a, 5); + assert!(g.prop_index_keys_eq(0, &PropertyValue::Int(1)).is_empty()); + + assert!(g.undelete_node(a, 5)); + assert_eq!(g.prop_index_keys_eq(0, &PropertyValue::Int(1)), &[a]); + assert_eq!(g.get_node(a).expect("node").deleted_lsn, u64::MAX); + } + + #[test] + fn test_undelete_node_wrong_lsn_is_noop() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], id_props(1), None, 1); + g.remove_node(a, 5); + assert!(!g.undelete_node(a, 99), "delete_lsn mismatch must no-op"); + assert!(g.prop_index_keys_eq(0, &PropertyValue::Int(1)).is_empty()); + } + + #[test] + fn test_resident_bytes_includes_prop_index() { + let mut g = MemGraph::new(1000); + let before = g.resident_bytes(); + for i in 0..20 { + g.add_node(smallvec![0], id_props(i), None, 1); + } + assert!( + g.resident_bytes() > before, + "resident_bytes must grow as the prop_index accumulates entries" + ); + } } diff --git a/src/graph/mod.rs b/src/graph/mod.rs index b78073f1d..01b92accd 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -1,9 +1,20 @@ //! Graph storage engine -- per-shard, segment-aligned property graph. //! //! Feature-gated under `graph` so the default build is unaffected. +//! +//! # Sharding model: single-shard graphs (W2-11) +//! +//! A named graph lives ENTIRELY on one shard — every GRAPH.* command routes +//! by hashing the graph name (`extract_primary_key` returns args[0]), so all +//! nodes, edges, segments, WAL records, and traversals for a graph are +//! shard-local. Redis-style hash tags in the graph name (`g{tenant1}`) +//! co-locate multiple graphs on one shard the same way they co-locate keys. +//! There is NO cross-shard traversal: an earlier scatter-gather scaffold +//! (`cross_shard.rs`: `handle_graph_traverse` + a `GraphTraverse` SPSC +//! message) never had a coordinator that sent it and was deleted — partition +//! a workload by GRAPH, not within one. pub mod compaction; -pub mod cross_shard; pub mod csr; pub mod cypher; pub mod fasthash; @@ -20,6 +31,7 @@ pub mod segment; pub mod simd; pub mod stats; pub mod store; +pub mod text_index; pub mod traversal; pub mod traversal_guard; pub mod types; @@ -27,10 +39,6 @@ pub mod view; pub mod visibility; pub mod wal; -pub use cross_shard::{ - DEFAULT_CROSS_SHARD_DEPTH_LIMIT, TraversalShardResult, graph_has_hash_tag, - handle_graph_traverse, parse_traverse_response, -}; pub use csr::{CsrSegment, CsrStorage, MmapCsrSegment}; pub use cypher::{CypherError, CypherQuery, is_read_only, parse_cypher}; pub use hybrid::{ diff --git a/src/graph/recovery.rs b/src/graph/recovery.rs index 586f3df78..91de6d905 100644 --- a/src/graph/recovery.rs +++ b/src/graph/recovery.rs @@ -13,31 +13,13 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; +use bytes::Bytes; + use crate::graph::csr::{CsrError, CsrStorage}; use crate::graph::manifest::GraphManifest; -use crate::graph::memgraph::MemGraph; use crate::graph::segment::GraphSegmentList; use crate::graph::store::GraphStore; -/// Compute the NodeKey index-space watermark for a set of loaded CSR -/// segments: one past the largest SlotMap index component among all -/// persisted `NodeMeta::external_id`s. Passed to `MemGraph::with_id_offset` -/// so that replay (or any post-recovery write) can never mint a NodeKey -/// that aliases a persisted external_id -- see `MemGraph::with_id_offset` -/// for the full soundness argument. Zero (no offset) when there are no -/// loaded segments -- there is nothing to alias against. -fn node_id_watermark(segments: &[Arc]) -> u32 { - segments - .iter() - .flat_map(|seg| seg.node_meta().iter()) - // KeyData::as_ffi() packs the SlotMap index in the low 32 bits - // (version occupies the high 32 bits) -- `as u32` truncation - // extracts exactly that index component. - .map(|meta| meta.external_id as u32) - .max() - .map_or(0, |max_idx| max_idx.saturating_add(1)) -} - /// Result of graph recovery for a single shard. pub struct GraphRecoveryResult { /// The recovered GraphStore with loaded segments. @@ -167,19 +149,24 @@ pub fn recover_graph_store( } } - // Inject loaded segments into the graph's segment holder, and - // fast-forward the mutable tier's key allocator (P0 fix: graph - // NodeKey aliasing across restart). At this point `graph.segments` - // still holds the pristine fresh MemGraph `create_graph` built via - // `GraphStore::load_metadata` -- nothing has written to it yet, so - // it is safe to replace outright rather than merge. + // Inject loaded segments into the graph's segment holder and restore + // the write buffer's id-allocation floors: the manifest cursors + // (authoritative when present; 0 in pre-cursor manifests) plus every + // frozen external_id (covers manifests written before cursors + // existed), so fresh post-recovery inserts can never alias a frozen + // row or a WAL-replayed id. if let Some(graph) = store.get_graph_mut(graph_name.as_bytes()) { - let watermark = node_id_watermark(&loaded_segments); + graph + .write_buf + .restore_id_cursors(manifest.next_node_id, manifest.next_edge_id); + for seg in &loaded_segments { + for nm in seg.node_meta() { + graph.write_buf.ensure_node_id_floor(nm.external_id); + } + } + let current = graph.segments.load(); graph.segments.swap(GraphSegmentList { - mutable: Some(Arc::new(MemGraph::with_id_offset( - graph.edge_threshold, - watermark, - ))), + mutable: current.mutable.clone(), immutable: loaded_segments, }); } @@ -231,8 +218,15 @@ pub fn save_graph_store( } } - // Write manifest. - let manifest = GraphManifest::from_segments(&graph_name, &segments.immutable, &base_dir); + // Write manifest (including the write buffer's id-allocation + // cursors, so recovery resumes allocation past every id ever + // handed out even if the WAL was truncated). + let manifest = GraphManifest::from_segments( + &graph_name, + &segments.immutable, + &base_dir, + graph.write_buf.id_cursors(), + ); let manifest_path = graph_data_dir.join("manifest.json"); manifest.save(&manifest_path)?; } @@ -245,10 +239,73 @@ pub fn shard_graph_dir(persistence_dir: &Path, shard_id: usize) -> PathBuf { persistence_dir.join(format!("shard_{shard_id}")) } +/// Snapshot the graph store as part of a WAL v3 checkpoint (Bug B of the +/// 2026-07 durability P0: the checkpoint advances the WAL replay floor and +/// recycles segments, so every graph record it covers must be materialized +/// on disk FIRST or a crash loses the graph permanently). +/// +/// Freezes each graph's write buffer into an immutable CSR segment (the +/// mutable tier is never serialized directly — freeze is the only path to +/// disk), stamps `snapshot_lsn` (the WAL LSN this snapshot covers; recovery +/// skips graph records at or below it), and persists segments + manifests + +/// metadata. +/// +/// Returns `true` when the checkpoint may proceed (snapshot persisted, or +/// nothing to do). Returns `false` on save failure — the caller MUST abort +/// the checkpoint finalize so the control file keeps the old replay floor +/// and the WAL segments holding the graph records are not recycled. +/// +/// Called on the shard thread between mutations (single-threaded event +/// loop), so the snapshot is a consistent cut: every record `<= snapshot_lsn` +/// is in the freeze, every later record is not. +pub fn persist_graph_at_checkpoint( + store: &mut GraphStore, + persistence_dir: Option<&Path>, + shard_id: usize, + snapshot_lsn: u64, +) -> bool { + if !store.is_dirty() || store.graph_count() == 0 { + return true; + } + // No persistence dir (e.g. --appendonly no): graph writes carry no + // durability contract; let the checkpoint proceed. + let Some(dir) = persistence_dir else { + return true; + }; + + // Freeze every write buffer so the on-disk segments cover the mutable + // tier. Graphs whose buffer is empty (or holds only cross-tier delta + // edges) skip the freeze — freeze_and_compact returns false without + // pushing an empty segment. + let names: Vec = store.list_graphs().into_iter().cloned().collect(); + for name in &names { + let lsn = store.allocate_lsn(); + if let Some(graph) = store.get_graph_mut(name) { + graph.freeze_and_compact(lsn); + } + } + + store.set_snapshot_lsn(snapshot_lsn); + match save_graph_store(store, dir, shard_id) { + Ok(()) => { + store.clear_dirty(); + true + } + Err(e) => { + tracing::error!( + "Shard {shard_id}: checkpoint graph snapshot failed ({e}); \ + aborting checkpoint finalize to keep the WAL replay floor" + ); + false + } + } +} + #[cfg(test)] mod tests { use super::*; use bytes::Bytes; + use slotmap::Key; use smallvec::smallvec; use tempfile::TempDir; @@ -302,6 +359,84 @@ mod tests { assert_eq!(segs.immutable[0].edge_count(), 2); } + /// P0-2 stable ids: recovery must restore the id-allocation cursors so + /// post-restart inserts can never alias frozen external_ids — via the + /// manifest cursors AND (for pre-cursor manifests) the frozen + /// external_ids themselves. + #[test] + fn test_recover_restores_id_allocation_floor() { + let dir = TempDir::new().expect("tmpdir"); + let shard_id = 0; + + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"g"), 64_000, 10) + .expect("ok"); + let graph = store.get_graph_mut(b"g").expect("exists"); + + // Simulate a pre-crash session: nodes allocated from the write + // buffer, then frozen into a CSR segment. + let a = graph.write_buf.add_node(smallvec![0], smallvec![], None, 1); + let b = graph.write_buf.add_node(smallvec![1], smallvec![], None, 1); + graph.write_buf.add_edge(a, b, 0, 1.0, None, 2).expect("ok"); + let frozen = graph.write_buf.freeze().expect("ok"); + let csr = CsrSegment::from_frozen(frozen, 100).expect("ok"); + graph.segments.add_immutable(csr); + graph.write_buf.thaw(); + let saved_cursors = graph.write_buf.id_cursors(); + let max_frozen_id = a.data().as_ffi().max(b.data().as_ffi()); + + save_graph_store(&store, dir.path(), shard_id).expect("save ok"); + + let mut result = recover_graph_store(dir.path(), shard_id) + .expect("io ok") + .expect("result exists"); + let graph = result.store.get_graph_mut(b"g").expect("exists"); + + // Cursors restored to at least the saved values. + let (nn, ne) = graph.write_buf.id_cursors(); + assert!( + nn >= saved_cursors.0, + "node cursor {nn} < saved {}", + saved_cursors.0 + ); + assert!( + ne >= saved_cursors.1, + "edge cursor {ne} < saved {}", + saved_cursors.1 + ); + + // A fresh insert must not alias any frozen row. + let fresh = graph.write_buf.add_node(smallvec![9], smallvec![], None, 5); + assert!( + fresh.data().as_ffi() > max_frozen_id, + "fresh id {} aliases frozen tier (max frozen {})", + fresh.data().as_ffi(), + max_frozen_id + ); + + // Pre-cursor manifest fallback: zero cursors, floor comes from the + // frozen external_ids scan. + let manifest_path = dir + .path() + .join(format!("shard_{shard_id}/graph_g/manifest.json")); + let mut manifest = GraphManifest::load(&manifest_path).expect("load ok"); + manifest.next_node_id = 0; + manifest.next_edge_id = 0; + manifest.save(&manifest_path).expect("save ok"); + + let mut result = recover_graph_store(dir.path(), shard_id) + .expect("io ok") + .expect("result exists"); + let graph = result.store.get_graph_mut(b"g").expect("exists"); + let fresh = graph.write_buf.add_node(smallvec![9], smallvec![], None, 5); + assert!( + fresh.data().as_ffi() > max_frozen_id, + "pre-cursor fallback: fresh id {} aliases frozen tier", + fresh.data().as_ffi() + ); + } + #[test] fn test_recover_with_corrupted_segment() { let dir = TempDir::new().expect("tmpdir"); diff --git a/src/graph/replay.rs b/src/graph/replay.rs index 290fc906e..6a3a2b46c 100644 --- a/src/graph/replay.rs +++ b/src/graph/replay.rs @@ -40,6 +40,21 @@ enum GraphCommand { RemoveNode { graph_name: Bytes, node_id: u64 }, /// GRAPH.REMOVEEDGE RemoveEdge { graph_name: Bytes, edge_id: u64 }, + /// GRAPH.SETPROP (W2-9: + /// Cypher SET durability — replayed after inserts, before removes). + SetProp { + graph_name: Bytes, + entity_id: u64, + is_node: bool, + key: u16, + value: PropertyValue, + }, + /// GRAPH.SETLABEL