Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/api/consolidation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::registry::{AccessError, AppState};
use super::vocabulary::vocabulary_audit;
use super::{
AppJson, AppPath, ErrorCode, MAX_MATCH_LIMIT, access_error, clamp, deadline_exceeded, error,
not_found, ok,
not_found, ok, overlong,
};

/// The detector stamp (the `louvain-cc/1` precedent): fingerprints are
Expand Down Expand Up @@ -185,6 +185,12 @@ pub async fn audit_consolidation(
AppJson(request): AppJson<ConsolidationAuditRequest>,
) -> Response {
let started_at = Instant::now();
// Before the dedup: `dedup` folds only CONSECUTIVE repeats, so an
// alternating list would otherwise pass both guards below at any
// length the body cap admits.
if let Some(refusal) = overlong("checks", request.checks.len(), started_at) {
return refusal;
}
let mut checks: Vec<&str> = request.checks.iter().map(String::as_str).collect();
checks.dedup();
if checks.is_empty() {
Expand Down
47 changes: 47 additions & 0 deletions src/context/consolidation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,16 @@ impl Context {
object: self.concept_name(edge.object).to_string(),
weight: edge.sum / edge.count as f64,
count: edge.count,
// Zero-sum records stay chain-linked (only a
// retraction unlinks) but attest nothing —
// the same posture `sign_conflicts` takes,
// whose zero-sum sources sit on neither side
// of a dispute. Listing one here would hand
// the judge a source that never net-asserted
// the triple.
sources: self
.attribution_chain(edge.first_attribution)
.filter(|(_, record)| record.sum != 0.0)
.map(|(_, record)| self.source_name(record.source).to_string())
.collect(),
}
Expand Down Expand Up @@ -507,6 +515,45 @@ mod tests {
assert!(after.iter().all(|g| g.label != "杜氏"), "{after:?}");
}

/// A source that asserted a triple and then cancelled its own
/// assertion (net sum exactly zero, record still chain-linked —
/// only a retraction unlinks) attests nothing: the evidence rows
/// handed to the judge must not list it, the same posture
/// `sign_conflicts` already takes for its zero-sum records.
#[test]
fn contradiction_group_sources_exclude_a_source_whose_own_sum_cancelled_to_zero() {
let mut context = Context::default();
context
.associate_from("蔵A", "杜氏", "高瀬", 1.0, "old", None)
.unwrap();
context
.associate_from("蔵A", "杜氏", "青山", 1.0, "new", None)
.unwrap();
// 撤回記事 asserts 高瀬, then cancels itself with the exact
// negative — chain-linked at sum 0.0, attesting nothing.
context
.associate_from("蔵A", "杜氏", "高瀬", 2.0, "撤回記事", None)
.unwrap();
context
.associate_from("蔵A", "杜氏", "高瀬", -2.0, "撤回記事", None)
.unwrap();
// Tendency needs company to make 杜氏 a candidate.
context.associate("蔵B", "杜氏", "田中", 1.0).unwrap();

let groups = context.contradiction_groups(Deadline::unbounded()).unwrap();
let toji = groups.iter().find(|g| g.label == "杜氏").unwrap();
let takase = toji
.objects
.iter()
.find(|row| row.object == "高瀬")
.unwrap();
assert_eq!(
takase.sources,
vec!["old"],
"the cancelled-out source must not read as attesting: {takase:?}"
);
}

#[test]
fn sign_conflicts_split_the_dispute_by_source() {
let mut context = Context::default();
Expand Down
134 changes: 119 additions & 15 deletions src/context/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -276,19 +276,20 @@ impl Context {
// pathological image whose every edge points at one long shared
// chain walks that chain once per edge — O(edges × chain) —
// during a migration that holds the write lock and never yields.
let mut chains: HashMap<AttributionId, (u64, f64)> = HashMap::new();
let mut chains: HashMap<AttributionId, (u64, f64, f64)> = HashMap::new();
for legacy in &legacy_edges {
let (chain_len, attributed_sum) = match chains.get(&legacy.first_attribution) {
Some(&cached) => cached,
None => {
let computed = legacy_attribution_chain_len(
&legacy_attributions,
legacy.first_attribution,
)?;
chains.insert(legacy.first_attribution, computed);
computed
}
};
let (chain_len, attributed_sum, attributed_magnitude) =
match chains.get(&legacy.first_attribution) {
Some(&cached) => cached,
None => {
let computed = legacy_attribution_chain_len(
&legacy_attributions,
legacy.first_attribution,
)?;
chains.insert(legacy.first_attribution, computed);
computed
}
};
// An empty chain (first_attribution == NIL) is ambiguous in
// the legacy format: it means either an edge that was always
// sourceless or one that was fully retracted (weight zeroed
Expand Down Expand Up @@ -325,7 +326,12 @@ impl Context {
// in context.rs.
let count = if chain_len == 0 && legacy.weight == 0.0 {
0
} else if legacy.weight != attributed_sum {
} else if summation_gap_is_real(
legacy.weight - attributed_sum,
chain_len,
attributed_magnitude,
legacy.weight,
) {
chain_len + 1
} else {
chain_len.max(1)
Expand Down Expand Up @@ -731,12 +737,39 @@ fn checked_arena_str(arena: &[u8], offset: u32, len: u32) -> Result<&str, Corrup
/// trusting: this runs before `index_attributions` has ever looked at the
/// chain, so a hostile or truncated pre-v5 image must not send it out of
/// bounds or looping forever on a cycle.
/// Whether a gap between a legacy edge's cumulative weight and its
/// chain's re-summed total proves a sourceless call, or is only
/// summation rounding. Not an exact `!=`: the edge accumulated its
/// calls in chronological order, the chain re-adds the same values in
/// chain order, and two orderings of the same f64 additions can differ
/// by rounding alone (a large-magnitude cancellation makes it
/// visible). An exact comparison would read that noise as proof of a
/// phantom sourceless call — weight `retract_source` can then never
/// remove. The bound is first-order naive-summation error for both
/// orders; a REAL sourceless call under it goes uncredited, the same
/// accepted false negative as the migration's zero-weight case.
/// Residual limitation, also accepted: a magnitude that vanished from
/// BOTH sides before migration (a large weight asserted then
/// retracted — its record unlinked, its rounding footprint still in
/// the edge weight) leaves nothing here to bound against, so that
/// shape can still credit a phantom; no tolerance derivable from the
/// image can tell it from a real sourceless call.
fn summation_gap_is_real(gap: f64, chain_len: u64, magnitude: f64, edge_weight: f64) -> bool {
let tolerance = 2.0 * (chain_len as f64 + 1.0) * f64::EPSILON * (magnitude + edge_weight.abs());
gap.abs() > tolerance
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

fn legacy_attribution_chain_len(
attributions: &[LegacyAttributionRecord],
mut cursor: AttributionId,
) -> Result<(u64, f64), CorruptImage> {
) -> Result<(u64, f64, f64), CorruptImage> {
let mut len = 0u64;
let mut sum = 0.0f64;
// Magnitude alongside the sum: the caller's sourceless-gap test
// needs an error bound for the two accumulation orders it
// compares, and first-order summation error scales with Σ|w|,
// not with the (possibly cancelled-to-nothing) net sum.
let mut magnitude = 0.0f64;
let mut steps: usize = 0;
while cursor != NIL {
steps += 1;
Expand All @@ -750,9 +783,10 @@ fn legacy_attribution_chain_len(
.ok_or(CorruptImage("legacy attribution link is out of range"))?;
len += 1;
accumulate_saturating(&mut sum, record.weight);
accumulate_saturating(&mut magnitude, record.weight.abs());
cursor = record.next;
}
Ok((len, sum))
Ok((len, sum, magnitude))
}

/// Checks that one linked chain of edges is exactly `count` records long,
Expand Down Expand Up @@ -1458,6 +1492,76 @@ mod tests {
assert!(after.attributions.is_empty());
}

/// Pins the tolerance formula itself, `2·(chain_len+1)·ε·(magnitude
/// plus |edge_weight|)` under a strict `>`, against hardcoded
/// boundary values, so no factor can silently change scale: for
/// chain_len 2, magnitude 1.0, weight −0.5 the bound is exactly
/// 1.9984014443252818e-15, and the gap must EXCEED it (the bound
/// itself is still attributable to rounding).
#[test]
fn the_summation_gap_bound_sits_exactly_at_first_order_rounding_error() {
let bound = 1.9984014443252818e-15;
assert!(!summation_gap_is_real(bound, 2, 1.0, -0.5));
assert!(!summation_gap_is_real(-bound, 2, 1.0, -0.5));
assert!(summation_gap_is_real(
1.998_401_444_325_282e-15,
2,
1.0,
-0.5
));
assert!(summation_gap_is_real(
-1.998_401_444_325_282e-15,
2,
1.0,
-0.5
));
// Well clear of the bound on both sides.
assert!(!summation_gap_is_real(bound / 2.0, 2, 1.0, -0.5));
assert!(summation_gap_is_real(bound * 2.0, 2, 1.0, -0.5));
// An empty chain still tolerates rounding on the edge weight
// alone (2·1·ε·|w|), rather than collapsing to zero width.
assert!(!summation_gap_is_real(f64::EPSILON, 0, 0.0, 1.0));
assert!(summation_gap_is_real(3.0 * f64::EPSILON, 0, 0.0, 1.0));
}

/// A fully sourced edge whose chain regroups the same additions
/// differently than the edge's own chronological accumulation
/// (`(0.1⊕0.3)⊕0.1⊕0.1 = 0.6` vs `(0.1⊕0.1⊕0.1)⊕0.3 =
/// 0.6000000000000001`) must NOT read that 1-ulp gap as a phantom
/// sourceless call: the phantom's count can never be retracted, so
/// it would keep this edge alive after every real source retracts.
#[test]
fn migrating_a_pre_v5_image_does_not_read_summation_rounding_as_a_phantom_sourceless_call() {
let mut context = Context::default();
context
.associate_from("私", "好き", "りんご", 0.1, "A", None)
.unwrap();
context
.associate_from("私", "好き", "りんご", 0.3, "B", None)
.unwrap();
context
.associate_from("私", "好き", "りんご", 0.1, "A", None)
.unwrap();
context
.associate_from("私", "好き", "りんご", 0.1, "A", None)
.unwrap();

let v4 = context.to_bytes_as_version(4);
let mut restored = Context::from_bytes(&v4).expect("v4 image must load");

// One record per source in the downgraded chain — no phantom
// third contribution from the rounding gap.
assert_eq!(restored.recall("私")[0].count, 2);
assert_eq!(restored.retract_source("A"), Some(1));
assert_eq!(restored.retract_source("B"), Some(1));
assert_eq!(
restored.dead_edges(),
1,
"with every real source retracted the edge must die — a phantom \
credit would hold it alive on rounding noise alone"
);
}

#[test]
fn image_roundtrip_preserves_every_read_path() {
let mut context = Context::default();
Expand Down
4 changes: 2 additions & 2 deletions src/extract/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -300,10 +300,10 @@ impl Args {
}
},
"--source-id" => match rest.next() {
Some(id) if source_id.is_none() && !id.is_empty() => {
Some(id) if source_id.is_none() && !id.trim().is_empty() => {
source_id = Some(id.clone());
}
Some(id) if id.is_empty() => {
Some(id) if id.trim().is_empty() => {
return Err(crate::config::subcommand_usage_error(
"extract",
"--source-id must not be empty",
Expand Down
4 changes: 4 additions & 0 deletions src/extract/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3699,6 +3699,10 @@ fn runbook_flags_parse_and_their_contradictions_are_usage_errors() {
let mut empty = base.to_vec();
empty.extend(["--source-id", "", "doc.md"]);
assert!(matches!(parse(&empty), Err(2)));
// Whitespace-only is the same emptiness — trimmed like --tag's.
let mut blank = base.to_vec();
blank.extend(["--source-id", " ", "doc.md"]);
assert!(matches!(parse(&blank), Err(2)));
let mut bad_date = base.to_vec();
bad_date.extend(["--date", "yesterday", "doc.md"]);
assert!(matches!(parse(&bad_date), Err(2)));
Expand Down
8 changes: 4 additions & 4 deletions src/mcp/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -620,8 +620,8 @@ pub(super) fn tool_definitions() -> Vec<Value> {
"limit": { "type": "integer", "minimum": 0, "description": "default 5" },
"semantic_floor": { "type": "number", "description": "one-call override of the vector lane's cosine floor (0-1); floors only the semantic lane — BM25-only hits still return" },
"tags": { "type": "array", "items": { "type": "string" }, "description": "only sources carrying at least one of these tags may answer" },
"since": { "type": "integer", "description": "only sources whose date ?? stored_at is at or after this (epoch seconds)" },
"until": { "type": "integer", "description": "only sources whose date ?? stored_at is strictly before this (epoch seconds)" }
"since": { "type": "integer", "minimum": 0, "description": "only sources whose date ?? stored_at is at or after this (epoch seconds)" },
"until": { "type": "integer", "minimum": 0, "description": "only sources whose date ?? stored_at is strictly before this (epoch seconds)" }
}),
&["query"],
),
Expand Down Expand Up @@ -696,8 +696,8 @@ pub(super) fn tool_definitions() -> Vec<Value> {
"limit": { "type": "integer", "minimum": 0, "description": "the search call being explained (default 5)" },
"semantic_floor": { "type": "number", "description": "the floor override of the search call being explained — pass the same value" },
"tags": { "type": "array", "items": { "type": "string" }, "description": "the tag filter of the search call being explained — pass the same values" },
"since": { "type": "integer", "description": "the time window's inclusive start (epoch seconds) of the search call being explained" },
"until": { "type": "integer", "description": "the time window's exclusive end (epoch seconds) of the search call being explained" }
"since": { "type": "integer", "minimum": 0, "description": "the time window's inclusive start (epoch seconds) of the search call being explained" },
"until": { "type": "integer", "minimum": 0, "description": "the time window's exclusive end (epoch seconds) of the search call being explained" }
}),
&["context", "query", "source"],
),
Expand Down
41 changes: 41 additions & 0 deletions src/registry/replication.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,18 @@ impl AppState {
// old bytes unreachable (see `EntryInner::cache_identity`).
inner.invalidate_cache_identity();
inner.load_failure = None;
// Re-stat both WAL gauges: on a replica the bytes arrive as
// tailed file copies, never through the writer's live
// increments, and `ensure_hot`'s own re-stat runs only for
// pinned entries below (or on the next local read) — without
// this, a cold unpinned context the tailer keeps growing
// understates `taguru_wal_bytes` indefinitely.
inner.wal_bytes = std::fs::metadata(wal_path(&self.0.data_dir, &stem))
.map(|meta| meta.len())
.unwrap_or(0);
inner.passages_wal_bytes = std::fs::metadata(passages_wal_path(&self.0.data_dir, &stem))
.map(|meta| meta.len())
.unwrap_or(0);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
if matches!(inner.slot, Slot::Hot(_)) {
inner.slot = Slot::Cold;
// The same bump eviction does: a flush that staged this
Expand Down Expand Up @@ -240,6 +252,35 @@ mod tests {
let _ = fs::remove_dir_all(dir);
}

/// On a replica the WAL grows by tailed file copies, never through
/// the writer's live byte accounting — the refresh must re-stat
/// both WAL gauges itself, or a cold unpinned context the tailer
/// keeps growing understates `taguru_wal_bytes` until some local
/// read happens to run `ensure_hot`.
#[test]
fn a_replica_refresh_restats_both_wal_gauges() {
let dir = scratch_dir("replica-refresh-wal-bytes");
let state = AppState::boot(dir.clone(), usize::MAX, None).unwrap();
state.create("sake", ContextMeta::default()).unwrap();
let stem = file_stem("sake");
// The tailer's shape: bytes land as plain file writes.
fs::write(wal_path(&dir, &stem), b"{\"tailed\":1}\n").unwrap();
fs::write(passages_wal_path(&dir, &stem), b"{\"tailed\":2}\n").unwrap();
state.replica_refresh("sake");
let entry = state.lookup("sake").unwrap();
let inner = entry.inner.read();
assert_eq!(
inner.wal_bytes,
fs::metadata(wal_path(&dir, &stem)).unwrap().len()
);
assert_eq!(
inner.passages_wal_bytes,
fs::metadata(passages_wal_path(&dir, &stem)).unwrap().len()
);
drop(inner);
let _ = fs::remove_dir_all(dir);
}

/// A tailed refresh of a HOT entry drops the slot and bumps the
/// image generation exactly like an eviction — a staged flush must
/// see the slot it captured is gone (vacuous on today's replica,
Expand Down
Loading
Loading