diff --git a/dist/src/admission-control.js b/dist/src/admission-control.js index 44733bb5..cee4e5aa 100644 --- a/dist/src/admission-control.js +++ b/dist/src/admission-control.js @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { DURABLE_CATEGORIES } from "./memory-categories.js"; import { parseSmartMetadata } from "./smart-metadata.js"; const DEFAULT_WEIGHTS = { utility: 0.1, @@ -322,12 +323,19 @@ Candidate memory: - Abstract: ${candidate.abstract} - Overview: ${candidate.overview} - Content: ${candidate.content} +- Grounding: ${candidate.grounding ?? "unknown (legacy payload, treat as real)"} +- Conversation register: ${candidate.conversationRegister ?? "unknown (legacy payload)"} Score future usefulness on a 0.0-1.0 scale. -Use higher scores for durable preferences, profile facts, reusable procedures, and long-lived project/entity state. +The memory system stores six categories: profile (user identity), preferences (user tendencies), entities (long-lived project/entity state), events (things that happened), cases (problem + solution pairs), and patterns (reusable procedures). + +Use higher scores for durable profile facts, preferences, entity state, patterns, and genuinely reusable cases. +Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. +Grounding rule: this candidate's own grounding tag already passed the deterministic pre-admission check (a "constructed" tag is rejected before this scoring ever runs), but a mistagged or legacy candidate can still describe a claim that is true only WITHIN a fiction — a persona's invented trait from roleplay, a game's rules or score, drafted fiction, a hypothetical, or sample data. If the candidate's own content shows such a constructed frame and its claim lives inside it rather than being a claim ABOUT the fiction (e.g. that a session/game happened), score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity is a claim ABOUT the fiction and may still score moderately. + Return JSON only: { "utility": 0.0, @@ -347,6 +355,22 @@ function buildReason(details) { export function scoreTypePrior(category, typePriors) { return clamp01(typePriors[category], DEFAULT_TYPE_PRIORS[category]); } +/** + * Grounding-aware type prior. The raw prior gives durable registers a large + * head start (default weights put 0.6 on this single feature); when the batch + * register says the conversation was fiction, that head start would launder a + * mislabeled in-fiction claim into profile/preferences. Cap the prior for + * durable categories at the events prior in that case; every other input + * keeps the raw prior untouched. + */ +export function scoreGroundedTypePrior(candidate, typePriors) { + const raw = scoreTypePrior(candidate.category, typePriors); + if (candidate.conversationRegister === "fiction" && + DURABLE_CATEGORIES.has(candidate.category)) { + return Math.min(raw, clamp01(typePriors.events, DEFAULT_TYPE_PRIORS.events)); + } + return raw; +} export function scoreConfidenceSupport(candidate, conversationText) { const candidateText = `${candidate.abstract}\n${candidate.content}`.trim(); const candidateTokens = tokenizeText(candidateText); @@ -437,6 +461,36 @@ export class AdmissionController { this.config = config; this.debugLog = debugLog; } + rejectConstructed(candidate, now) { + const featureScores = { + utility: 0, + confidence: 0, + novelty: 0, + recency: 0, + typePrior: 0, + }; + const reason = `Admission rejected (constructed-grounding candidate category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const audit = { + version: "amac-v1", + decision: "reject", + score: 0, + reason, + thresholds: { + reject: this.config.rejectThreshold, + admit: this.config.admitThreshold, + }, + weights: this.config.weights, + feature_scores: featureScores, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0, + evaluated_at: now, + grounding: candidate.grounding, + conversation_register: candidate.conversationRegister, + }; + this.debugLog(`memory-lancedb-pro: admission-control: decision=reject (constructed short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`); + return { decision: "reject", audit }; + } async loadRelevantMatches(candidate, candidateVector, scopeFilter) { if (!Array.isArray(candidateVector) || candidateVector.length === 0) { return []; @@ -453,12 +507,21 @@ export class AdmissionController { } async evaluate(params) { const now = params.now ?? Date.now(); + // Deterministic pre-admission short-circuit: a candidate tagged + // "constructed" is true only within the fiction, never about the real + // world — it must never be stored, in any category or register, no + // matter how any downstream score would blend. Reject before any LLM + // call. Candidates without grounding metadata (legacy payloads) fall + // through to normal scoring. + if (params.candidate.grounding === "constructed") { + return this.rejectConstructed(params.candidate, now); + } const relevantMatches = await this.loadRelevantMatches(params.candidate, params.candidateVector, params.scopeFilter); const utility = await scoreUtility(this.llm, this.config.utilityMode, params.candidate, params.conversationText); const confidence = scoreConfidenceSupport(params.candidate, params.conversationText); const novelty = scoreNoveltyFromMatches(params.candidateVector, relevantMatches); const recency = scoreRecencyGap(now, relevantMatches, this.config.recency.halfLifeDays); - const typePrior = scoreTypePrior(params.candidate.category, this.config.typePriors); + const typePrior = scoreGroundedTypePrior(params.candidate, this.config.typePriors); const featureScores = { utility: utility.score, confidence: confidence.score, @@ -502,6 +565,8 @@ export class AdmissionController { compared_existing_memory_ids: novelty.comparedIds, max_similarity: novelty.maxSimilarity, evaluated_at: now, + grounding: params.candidate.grounding, + conversation_register: params.candidate.conversationRegister, }; this.debugLog(`memory-lancedb-pro: admission-control: decision=${audit.decision} hint=${audit.hint ?? "n/a"} score=${audit.score.toFixed(3)} candidate=${JSON.stringify(params.candidate.abstract.slice(0, 80))}`); return { decision, hint, audit }; diff --git a/dist/src/extraction-prompts.js b/dist/src/extraction-prompts.js index b74c9b3d..132f6138 100644 --- a/dist/src/extraction-prompts.js +++ b/dist/src/extraction-prompts.js @@ -31,10 +31,10 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete fact/preference/decision/entity from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile detail, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. -- Atomic memory shape: each stored memory must read like one durable fact, preference, decision, entity state, event, case, or reusable pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. +- Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. - Length/distillation gate: if a candidate is longer than about 200 characters and reads like raw conversation instead of a distilled insight, rewrite it as a single factual statement before storing; if that is not possible, skip it. # Memory Classification @@ -66,6 +66,41 @@ ${conversationText} - "Encountered problem A, used solution B" -> cases (not events) - "General process for handling certain problems" -> patterns (not cases) +# Conversational Grounding + +First judge the register of the WHOLE conversation once, then tag each memory item independently. + +## Batch register (one judgment per extraction) + +Set the top-level "conversation_register" field: + +| conversation_register | Meaning | +|-----------------------|---------| +| "real" | An ordinary working/personal conversation about the actual user and the real world | +| "fiction" | The conversation is inside a constructed frame: a game in progress, roleplay, in-character dialogue, drafted fiction, a hypothetical scenario, or a sample-data exercise | +| "mixed" | A constructed frame AND genuine out-of-character content are interleaved | + +## Per-item grounding + +Grounding describes the truth-grounding of the ASSERTION ITSELF, not which register the conversation happened in. Ask: is this claim true about the real world, or true only inside the fiction? + +Tag every memory's "grounding" field: + +| grounding | Meaning | +|-----------|---------| +| "real" | The assertion is about the real world — INCLUDING an assertion ABOUT the fiction (e.g. "user and assistant played a one-round roleplay game where user was Admiral Vex" is a true statement about a real session, even though it describes fictional play). Also covers a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | The assertion holds only WITHIN the fiction — true in-character, in-game, or in-story, but not a fact about the real user or the real world (e.g. "user's favorite drink is plasma coffee", a persona's invented backstory, a game's score or rules) | + +One-line rule: **about-the-fiction is real; within-the-fiction is constructed.** + +Rules: +- Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. +- A session-scoped "events" note that the real participants engaged in a game, roleplay, or other fiction is a REAL assertion — it is a true statement about what happened in the real session, not a claim that lives inside the fiction. Extract it like any other events item, under its natural category, with grounding "real". +- Do NOT lift any in-character proposition — an invented rule, a score, a bet, a persona's claim, a fictional preference or trait — into profile, preferences, entities, cases, or patterns. Such claims are true only within the fiction; if you extract one at all, tag it "constructed" so it is never mistaken for a fact about the real user. +- A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. +- Self-consistency check before answering: an item asserting what happened in the real session (including a session that was itself a game) is "real"; an item asserting a claim that is only true inside the story/game/persona is "constructed". If your draft tags the session-summary note itself as "constructed", re-check — it almost certainly describes a real event and should be "real". +- If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. + # Three-Level Structure Each memory contains three levels: @@ -80,33 +115,85 @@ Each memory contains three levels: # Few-shot Examples -## profile +Each example is a full output batch, because register and grounding are judged together. + +## Ordinary working conversation (register "real", single memory) \`\`\`json { - "category": "profile", - "abstract": "User basic info: AI development engineer, 3 years LLM experience", - "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience." + "conversation_register": "real", + "memories": [ + { + "category": "cases", + "abstract": "LanceDB BigInt numeric handling issue", + "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" + } + ] } \`\`\` -## preferences +## Ordinary personal conversation (register "real", two memories) \`\`\`json { - "category": "preferences", - "abstract": "Python code style: No type hints, concise and direct", - "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments." + "conversation_register": "real", + "memories": [ + { + "category": "profile", + "abstract": "User basic info: AI development engineer, 3 years LLM experience", + "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" + }, + { + "category": "preferences", + "abstract": "Python code style: No type hints, concise and direct", + "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" + } + ] } \`\`\` -## cases +## Mid-game conversation (register "fiction", session note is real; canon is not extracted) +Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { - "category": "cases", - "abstract": "LanceDB BigInt numeric handling issue", - "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations." + "conversation_register": "fiction", + "memories": [ + { + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "real" + } + ] +} +\`\`\` +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. This session note is a true statement about a real session (the session happened), so it carries grounding "real" even though the batch register is "fiction" — about-the-fiction is real. + +## Game with a genuine out-of-character aside (register "mixed") +\`\`\`json +{ + "conversation_register": "mixed", + "memories": [ + { + "category": "events", + "abstract": "User mentioned their new laptop arrives Thursday", + "overview": "## Real-world aside\\n- Stated in passing during a game", + "content": "In the middle of the game the user mentioned, out of character, that their new laptop arrives Thursday.", + "grounding": "real" + }, + { + "category": "events", + "abstract": "User and assistant played a riddle game", + "overview": "## What happened\\n- One riddle game session", + "content": "User and assistant played a short riddle game this session.", + "grounding": "real" + } + ] } \`\`\` @@ -114,12 +201,14 @@ Each memory contains three levels: Return JSON: { + "conversation_register": "real|mixed|fiction", "memories": [ { "category": "profile|preferences|entities|events|cases|patterns", "abstract": "One-line index", "overview": "Structured Markdown summary", - "content": "Full narrative" + "content": "Full narrative", + "grounding": "real|constructed" } ] } @@ -127,9 +216,10 @@ Return JSON: Notes: - Output language should match the dominant language in the conversation - Only extract truly valuable personalized information -- If nothing worth recording, return {"memories": []} +- If nothing worth recording, return {"conversation_register": "real|mixed|fiction", "memories": []} - Maximum 5 memories per extraction -- Preferences should be aggregated by topic`; +- Preferences should be aggregated by topic +- Always set the top-level "conversation_register" field, and tag every memory's "grounding" field, per the Conversational Grounding rules above`; } export function buildDedupPrompt(candidateAbstract, candidateOverview, candidateContent, existingMemories) { return `Determine how to handle this candidate memory. diff --git a/dist/src/memory-categories.js b/dist/src/memory-categories.js index abba922e..75f7e96f 100644 --- a/dist/src/memory-categories.js +++ b/dist/src/memory-categories.js @@ -58,6 +58,19 @@ export const APPEND_ONLY_CATEGORIES = new Set([ "events", "cases", ]); +/** + * Durable categories: governs fiction-register batch enforcement (an + * in-fiction batch can never produce durable memories) and the batch + * contradiction check. Per-item grounding "constructed" is dropped + * unconditionally in every category, so this set does not gate that rule. + */ +export const DURABLE_CATEGORIES = new Set([ + "profile", + "preferences", + "entities", + "cases", + "patterns", +]); /** Validate and normalize a category string. */ export function normalizeCategory(raw) { const lower = raw.toLowerCase().trim(); diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 1bd45d99..0833c2b2 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -7,7 +7,7 @@ */ import { buildExtractionPrompt, buildDedupPrompt, buildMergePrompt, } from "./extraction-prompts.js"; import { AdmissionController, } from "./admission-control.js"; -import { ALWAYS_MERGE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; +import { ALWAYS_MERGE_CATEGORIES, DURABLE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; import { isMetaFrustrationNoise, isNoise } from "./noise-filter.js"; import { appendRelation, buildSmartMetadata, deriveFactKey, parseSmartMetadata, stringifySmartMetadata, parseSupportInfo, updateSupportStats, } from "./smart-metadata.js"; import { isUserMdExclusiveMemory, } from "./workspace-boundary.js"; @@ -154,6 +154,28 @@ export function stripEnvelopeMetadata(text) { cleaned = cleaned.replace(/\n{3,}/g, "\n\n"); return cleaned.trim(); } +function globToRegExp(glob) { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + return new RegExp(`^${escaped}$`); +} +/** + * Resolve the extraction policy for a scope against a scope-glob -> mode map. + * Exact-string entries take priority over glob entries; an unmatched scope + * (or an absent policy map) defaults to "full". + */ +export function resolveExtractionPolicy(scope, policy) { + if (!policy) + return "full"; + if (Object.prototype.hasOwnProperty.call(policy, scope)) { + return policy[scope]; + } + for (const [glob, mode] of Object.entries(policy)) { + if (glob.includes("*") && globToRegExp(glob).test(scope)) { + return mode; + } + } + return "full"; +} // ============================================================================ // Constants // ============================================================================ @@ -244,15 +266,29 @@ export class SmartExtractor { ? options.scopeFilter : [targetScope]; const agentId = options.agentId; + // Option C: scope-glob extraction policy — "none" skips extraction + // entirely, with zero LLM calls, before grounding is ever considered. + const policyMode = resolveExtractionPolicy(targetScope, this.config.extractionPolicy); + if (policyMode === "none") { + this.log(`memory-pro: smart-extractor: extraction policy "none" for scope ${targetScope}, skipping extraction`); + return stats; + } // Step 1: LLM extraction - const extraction = await this.extractCandidates(conversationText); + const extraction = await this.extractCandidates(conversationText, policyMode); const candidates = extraction.candidates; if (candidates.length === 0) { this.log("memory-pro: smart-extractor: no memories extracted"); - if (extraction.status === "ok") { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); } + else if (extraction.status === "ok") { + // The model DID emit candidates; validation dropped or demoted them + // all. That is a policy verdict about those candidates, not evidence + // the conversation is noise — learning it as noise would pre-filter + // similar real content away from future extractions. + this.log(`memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`); + } else { this.debugLog(`memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`); } @@ -515,7 +551,7 @@ export class SmartExtractor { /** * Call LLM to extract candidate memories from conversation text. */ - async extractCandidates(conversationText) { + async extractCandidates(conversationText, policyMode = "full") { const maxChars = this.config.extractMaxChars ?? 8000; const truncated = conversationText.length > maxChars ? conversationText.slice(-maxChars) @@ -536,11 +572,24 @@ export class SmartExtractor { return { status: "malformed", candidates: [] }; } this.debugLog(`memory-lancedb-pro: smart-extractor: extract-candidates raw memories=${result.memories.length}`); + // Batch-level register signal, judged once per extraction. The model + // classifies whole sessions far more reliably than it self-tags single + // items, so the register deterministically overrides per-item grounding + // wobble below. Missing/unrecognized values fail toward scrutiny + // ("mixed"), never toward open. + const rawRegister = typeof result.conversation_register === "string" + ? result.conversation_register.toLowerCase().trim() + : ""; + const conversationRegister = rawRegister === "real" || rawRegister === "fiction" ? rawRegister : "mixed"; // Validate and normalize candidates const candidates = []; let invalidCategoryCount = 0; let shortAbstractCount = 0; let noiseAbstractCount = 0; + let policyDroppedCount = 0; + let constructedDroppedCount = 0; + let fictionRegisterDroppedCount = 0; + let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -567,10 +616,67 @@ export class SmartExtractor { this.debugLog(`memory-lancedb-pro: smart-extractor: dropping candidate due to noise abstract category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); continue; } - candidates.push({ category, abstract, overview, content }); + // Option C: scope policy restricts extraction to episodic-only, + // independent of grounding — checked before the grounding filter below. + if (policyMode === "episodic-only" && category !== "events") { + policyDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping candidate due to episodic-only extraction policy category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + // Option A / v3: grounding-aware filter. Missing/non-string/unrecognized + // per-item values fail open to "real" so a model that ignores the field + // can't break extraction. Grounding describes the truth-grounding of the + // ASSERTION itself: "real" includes an assertion ABOUT a fiction/game + // session (e.g. that it happened); "constructed" is a claim true only + // WITHIN the fiction. A constructed-tagged candidate is never stored, + // in any category or register — there is no per-extraction cap anymore. + const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; + const grounding = rawGrounding === "constructed" ? "constructed" : "real"; + if (rawGrounding === "constructed") { + batchHasConstructedTag = true; + } + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override). + if (conversationRegister === "fiction" && DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + // Grounding enforcement: a constructed assertion is true only within + // the fiction, never about the real world — never stored, regardless + // of category or register. + if (grounding === "constructed") { + constructedDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); + } + // Batch contradiction check: when the batch shows evidence of a + // constructed frame (register not "real" AND at least one item the model + // itself tagged constructed), surviving real-tagged durables in the same + // batch are the classic mislabel shape — fiction items where only the + // grounding tag wobbled. Demote them deterministically, no extra LLM call. + let contradictionDemotedCount = 0; + if (conversationRegister !== "real" && batchHasConstructedTag) { + for (let i = candidates.length - 1; i >= 0; i--) { + const candidate = candidates[i]; + if (DURABLE_CATEGORIES.has(candidate.category)) { + contradictionDemotedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: demoting real-tagged durable from ${conversationRegister}-register batch with constructed siblings (batch contradiction) category=${candidate.category} abstract=${JSON.stringify(candidate.abstract.slice(0, 120))}`); + candidates.splice(i, 1); + } + } + } + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log(`memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`); } - this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}`); - return { status: "ok", candidates }; + this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`); + return { status: "ok", candidates, rawCandidateCount: result.memories.length }; } // -------------------------------------------------------------------------- // Step 2: Dedup + Persist @@ -1139,6 +1245,12 @@ export class SmartExtractor { suppressed_until_turn: 0, memory_temporal_type: classifyTemporal(classifyText), valid_until: inferExpiry(classifyText), + // Grounding audit trail: which self-tag and batch register this + // memory was admitted under. Absent on legacy payloads. + ...(candidate.grounding ? { grounding: candidate.grounding } : {}), + ...(candidate.conversationRegister + ? { conversation_register: candidate.conversationRegister } + : {}), ...(admissionAudit ? { admission_audit: JSON.stringify(admissionAudit) } : {}), })); return { diff --git a/package.json b/package.json index 8260b2e8..c43770dc 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/extraction-grounding-register.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", "test:llm-clients-and-auth": "node scripts/run-ci-tests.mjs --group llm-clients-and-auth", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index dce4388f..befa346f 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -110,6 +110,7 @@ export const CI_TEST_MANIFEST = [ // Delete/delete-bulk must synchronously invalidate in-process reflection read caches { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/reflection-mapped-rows-admission.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/extraction-grounding-register.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { diff --git a/src/admission-control.ts b/src/admission-control.ts index eee44d35..3e7ea97e 100644 --- a/src/admission-control.ts +++ b/src/admission-control.ts @@ -1,6 +1,6 @@ import { join } from "node:path"; import type { LlmClient } from "./llm-client.js"; -import type { CandidateMemory, MemoryCategory } from "./memory-categories.js"; +import { DURABLE_CATEGORIES, type CandidateMemory, type MemoryCategory } from "./memory-categories.js"; import type { MemorySearchResult, MemoryStore } from "./store.js"; import { parseSmartMetadata } from "./smart-metadata.js"; @@ -70,6 +70,10 @@ export interface AdmissionAuditRecord { compared_existing_memory_ids: string[]; max_similarity: number; evaluated_at: number; + /** Candidate's extraction-time grounding self-tag; absent on legacy payloads. */ + grounding?: "real" | "constructed"; + /** Batch register the candidate was extracted under; absent on legacy payloads. */ + conversation_register?: "real" | "mixed" | "fiction"; } export interface AdmissionEvaluation { @@ -474,12 +478,19 @@ Candidate memory: - Abstract: ${candidate.abstract} - Overview: ${candidate.overview} - Content: ${candidate.content} +- Grounding: ${candidate.grounding ?? "unknown (legacy payload, treat as real)"} +- Conversation register: ${candidate.conversationRegister ?? "unknown (legacy payload)"} Score future usefulness on a 0.0-1.0 scale. -Use higher scores for durable preferences, profile facts, reusable procedures, and long-lived project/entity state. +The memory system stores six categories: profile (user identity), preferences (user tendencies), entities (long-lived project/entity state), events (things that happened), cases (problem + solution pairs), and patterns (reusable procedures). + +Use higher scores for durable profile facts, preferences, entity state, patterns, and genuinely reusable cases. +Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. +Grounding rule: this candidate's own grounding tag already passed the deterministic pre-admission check (a "constructed" tag is rejected before this scoring ever runs), but a mistagged or legacy candidate can still describe a claim that is true only WITHIN a fiction — a persona's invented trait from roleplay, a game's rules or score, drafted fiction, a hypothetical, or sample data. If the candidate's own content shows such a constructed frame and its claim lives inside it rather than being a claim ABOUT the fiction (e.g. that a session/game happened), score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity is a claim ABOUT the fiction and may still score moderately. + Return JSON only: { "utility": 0.0, @@ -512,6 +523,28 @@ export function scoreTypePrior( return clamp01(typePriors[category], DEFAULT_TYPE_PRIORS[category]); } +/** + * Grounding-aware type prior. The raw prior gives durable registers a large + * head start (default weights put 0.6 on this single feature); when the batch + * register says the conversation was fiction, that head start would launder a + * mislabeled in-fiction claim into profile/preferences. Cap the prior for + * durable categories at the events prior in that case; every other input + * keeps the raw prior untouched. + */ +export function scoreGroundedTypePrior( + candidate: CandidateMemory, + typePriors: AdmissionTypePriors, +): number { + const raw = scoreTypePrior(candidate.category, typePriors); + if ( + candidate.conversationRegister === "fiction" && + DURABLE_CATEGORIES.has(candidate.category) + ) { + return Math.min(raw, clamp01(typePriors.events, DEFAULT_TYPE_PRIORS.events)); + } + return raw; +} + export function scoreConfidenceSupport( candidate: CandidateMemory, conversationText: string, @@ -635,6 +668,42 @@ export class AdmissionController { private readonly debugLog: (msg: string) => void = () => {}, ) {} + private rejectConstructed( + candidate: CandidateMemory, + now: number, + ): AdmissionEvaluation { + const featureScores: AdmissionFeatureScores = { + utility: 0, + confidence: 0, + novelty: 0, + recency: 0, + typePrior: 0, + }; + const reason = `Admission rejected (constructed-grounding candidate category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const audit: AdmissionAuditRecord = { + version: "amac-v1", + decision: "reject", + score: 0, + reason, + thresholds: { + reject: this.config.rejectThreshold, + admit: this.config.admitThreshold, + }, + weights: this.config.weights, + feature_scores: featureScores, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0, + evaluated_at: now, + grounding: candidate.grounding, + conversation_register: candidate.conversationRegister, + }; + this.debugLog( + `memory-lancedb-pro: admission-control: decision=reject (constructed short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`, + ); + return { decision: "reject", audit }; + } + private async loadRelevantMatches( candidate: CandidateMemory, candidateVector: number[], @@ -671,6 +740,17 @@ export class AdmissionController { now?: number; }): Promise { const now = params.now ?? Date.now(); + + // Deterministic pre-admission short-circuit: a candidate tagged + // "constructed" is true only within the fiction, never about the real + // world — it must never be stored, in any category or register, no + // matter how any downstream score would blend. Reject before any LLM + // call. Candidates without grounding metadata (legacy payloads) fall + // through to normal scoring. + if (params.candidate.grounding === "constructed") { + return this.rejectConstructed(params.candidate, now); + } + const relevantMatches = await this.loadRelevantMatches( params.candidate, params.candidateVector, @@ -686,7 +766,7 @@ export class AdmissionController { const confidence = scoreConfidenceSupport(params.candidate, params.conversationText); const novelty = scoreNoveltyFromMatches(params.candidateVector, relevantMatches); const recency = scoreRecencyGap(now, relevantMatches, this.config.recency.halfLifeDays); - const typePrior = scoreTypePrior(params.candidate.category, this.config.typePriors); + const typePrior = scoreGroundedTypePrior(params.candidate, this.config.typePriors); const featureScores: AdmissionFeatureScores = { utility: utility.score, @@ -737,6 +817,8 @@ export class AdmissionController { compared_existing_memory_ids: novelty.comparedIds, max_similarity: novelty.maxSimilarity, evaluated_at: now, + grounding: params.candidate.grounding, + conversation_register: params.candidate.conversationRegister, }; this.debugLog( diff --git a/src/extraction-prompts.ts b/src/extraction-prompts.ts index 93a787f4..85962c55 100644 --- a/src/extraction-prompts.ts +++ b/src/extraction-prompts.ts @@ -35,10 +35,10 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete fact/preference/decision/entity from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile detail, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. -- Atomic memory shape: each stored memory must read like one durable fact, preference, decision, entity state, event, case, or reusable pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. +- Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. - Length/distillation gate: if a candidate is longer than about 200 characters and reads like raw conversation instead of a distilled insight, rewrite it as a single factual statement before storing; if that is not possible, skip it. # Memory Classification @@ -70,6 +70,41 @@ ${conversationText} - "Encountered problem A, used solution B" -> cases (not events) - "General process for handling certain problems" -> patterns (not cases) +# Conversational Grounding + +First judge the register of the WHOLE conversation once, then tag each memory item independently. + +## Batch register (one judgment per extraction) + +Set the top-level "conversation_register" field: + +| conversation_register | Meaning | +|-----------------------|---------| +| "real" | An ordinary working/personal conversation about the actual user and the real world | +| "fiction" | The conversation is inside a constructed frame: a game in progress, roleplay, in-character dialogue, drafted fiction, a hypothetical scenario, or a sample-data exercise | +| "mixed" | A constructed frame AND genuine out-of-character content are interleaved | + +## Per-item grounding + +Grounding describes the truth-grounding of the ASSERTION ITSELF, not which register the conversation happened in. Ask: is this claim true about the real world, or true only inside the fiction? + +Tag every memory's "grounding" field: + +| grounding | Meaning | +|-----------|---------| +| "real" | The assertion is about the real world — INCLUDING an assertion ABOUT the fiction (e.g. "user and assistant played a one-round roleplay game where user was Admiral Vex" is a true statement about a real session, even though it describes fictional play). Also covers a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | The assertion holds only WITHIN the fiction — true in-character, in-game, or in-story, but not a fact about the real user or the real world (e.g. "user's favorite drink is plasma coffee", a persona's invented backstory, a game's score or rules) | + +One-line rule: **about-the-fiction is real; within-the-fiction is constructed.** + +Rules: +- Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. +- A session-scoped "events" note that the real participants engaged in a game, roleplay, or other fiction is a REAL assertion — it is a true statement about what happened in the real session, not a claim that lives inside the fiction. Extract it like any other events item, under its natural category, with grounding "real". +- Do NOT lift any in-character proposition — an invented rule, a score, a bet, a persona's claim, a fictional preference or trait — into profile, preferences, entities, cases, or patterns. Such claims are true only within the fiction; if you extract one at all, tag it "constructed" so it is never mistaken for a fact about the real user. +- A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. +- Self-consistency check before answering: an item asserting what happened in the real session (including a session that was itself a game) is "real"; an item asserting a claim that is only true inside the story/game/persona is "constructed". If your draft tags the session-summary note itself as "constructed", re-check — it almost certainly describes a real event and should be "real". +- If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. + # Three-Level Structure Each memory contains three levels: @@ -84,33 +119,85 @@ Each memory contains three levels: # Few-shot Examples -## profile +Each example is a full output batch, because register and grounding are judged together. + +## Ordinary working conversation (register "real", single memory) \`\`\`json { - "category": "profile", - "abstract": "User basic info: AI development engineer, 3 years LLM experience", - "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience." + "conversation_register": "real", + "memories": [ + { + "category": "cases", + "abstract": "LanceDB BigInt numeric handling issue", + "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" + } + ] } \`\`\` -## preferences +## Ordinary personal conversation (register "real", two memories) \`\`\`json { - "category": "preferences", - "abstract": "Python code style: No type hints, concise and direct", - "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments." + "conversation_register": "real", + "memories": [ + { + "category": "profile", + "abstract": "User basic info: AI development engineer, 3 years LLM experience", + "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" + }, + { + "category": "preferences", + "abstract": "Python code style: No type hints, concise and direct", + "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" + } + ] } \`\`\` -## cases +## Mid-game conversation (register "fiction", session note is real; canon is not extracted) +Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { - "category": "cases", - "abstract": "LanceDB BigInt numeric handling issue", - "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations." + "conversation_register": "fiction", + "memories": [ + { + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "real" + } + ] +} +\`\`\` +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. This session note is a true statement about a real session (the session happened), so it carries grounding "real" even though the batch register is "fiction" — about-the-fiction is real. + +## Game with a genuine out-of-character aside (register "mixed") +\`\`\`json +{ + "conversation_register": "mixed", + "memories": [ + { + "category": "events", + "abstract": "User mentioned their new laptop arrives Thursday", + "overview": "## Real-world aside\\n- Stated in passing during a game", + "content": "In the middle of the game the user mentioned, out of character, that their new laptop arrives Thursday.", + "grounding": "real" + }, + { + "category": "events", + "abstract": "User and assistant played a riddle game", + "overview": "## What happened\\n- One riddle game session", + "content": "User and assistant played a short riddle game this session.", + "grounding": "real" + } + ] } \`\`\` @@ -118,12 +205,14 @@ Each memory contains three levels: Return JSON: { + "conversation_register": "real|mixed|fiction", "memories": [ { "category": "profile|preferences|entities|events|cases|patterns", "abstract": "One-line index", "overview": "Structured Markdown summary", - "content": "Full narrative" + "content": "Full narrative", + "grounding": "real|constructed" } ] } @@ -131,9 +220,10 @@ Return JSON: Notes: - Output language should match the dominant language in the conversation - Only extract truly valuable personalized information -- If nothing worth recording, return {"memories": []} +- If nothing worth recording, return {"conversation_register": "real|mixed|fiction", "memories": []} - Maximum 5 memories per extraction -- Preferences should be aggregated by topic`; +- Preferences should be aggregated by topic +- Always set the top-level "conversation_register" field, and tag every memory's "grounding" field, per the Conversational Grounding rules above`; } export function buildDedupPrompt( diff --git a/src/memory-categories.ts b/src/memory-categories.ts index 809c0327..9158d3c3 100644 --- a/src/memory-categories.ts +++ b/src/memory-categories.ts @@ -80,12 +80,41 @@ export const APPEND_ONLY_CATEGORIES = new Set([ /** Memory tier levels for lifecycle management. */ export type MemoryTier = "core" | "working" | "peripheral"; +/** Per-candidate conversational grounding self-tag from extraction. */ +export type CandidateGrounding = "real" | "constructed"; + +/** + * Batch-level register judgment for the whole extraction input: + * "real" ordinary conversation, "fiction" an in-character/game/roleplay + * frame, "mixed" both interleaved. Judged once per extraction batch — + * more stable than the per-item grounding tags. + */ +export type ConversationRegister = "real" | "mixed" | "fiction"; + +/** + * Durable categories: governs fiction-register batch enforcement (an + * in-fiction batch can never produce durable memories) and the batch + * contradiction check. Per-item grounding "constructed" is dropped + * unconditionally in every category, so this set does not gate that rule. + */ +export const DURABLE_CATEGORIES = new Set([ + "profile", + "preferences", + "entities", + "cases", + "patterns", +]); + /** A candidate memory extracted from conversation by LLM. */ export type CandidateMemory = { category: MemoryCategory; abstract: string; // L0: one-sentence index overview: string; // L1: structured markdown summary content: string; // L2: full narrative + /** Absent on legacy payloads: treat as "real" (fail open per item). */ + grounding?: CandidateGrounding; + /** Batch register the candidate was extracted under; absent on legacy payloads. */ + conversationRegister?: ConversationRegister; }; /** Dedup decision from LLM. */ diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index b74ac818..17223f1b 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -21,12 +21,15 @@ import { type AdmissionRejectionAuditEntry, } from "./admission-control.js"; import { + type CandidateGrounding, type CandidateMemory, + type ConversationRegister, type DedupDecision, type DedupResult, type ExtractionStats, type MemoryCategory, ALWAYS_MERGE_CATEGORIES, + DURABLE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, MEMORY_CATEGORIES, @@ -64,9 +67,12 @@ type PendingSupersedeInvalidation = { // Discriminates WHY extractCandidates() came back empty, so the caller can // tell a genuine "LLM found nothing" verdict from a gateway/model failure or -// a malformed response shape — only the former is a real noise signal. +// a malformed response shape. rawCandidateCount carries how many items the +// model emitted BEFORE validation: only "ok" with a raw count of zero is a +// real noise signal — a validation-emptied batch is a policy verdict about +// candidates the model did find, not evidence the conversation is noise. type ExtractCandidatesResult = - | { status: "ok"; candidates: CandidateMemory[] } + | { status: "ok"; candidates: CandidateMemory[]; rawCandidateCount: number } | { status: "llm_failure"; candidates: [] } | { status: "malformed"; candidates: [] }; @@ -234,6 +240,45 @@ export function stripEnvelopeMetadata(text: string): string { return cleaned.trim(); } +// ============================================================================ +// Extraction Policy (Option C — scope-glob knob) +// ============================================================================ + +/** + * Operator-facing extraction policy for a scope: + * - "full" (default): today's behavior, unchanged. + * - "episodic-only": only "events"-class candidates are kept, regardless of + * grounding — a blunt backstop for scopes known to be pure play/roleplay. + * - "none": extraction is skipped entirely, with zero LLM calls. + */ +export type ExtractionPolicyMode = "full" | "episodic-only" | "none"; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + return new RegExp(`^${escaped}$`); +} + +/** + * Resolve the extraction policy for a scope against a scope-glob -> mode map. + * Exact-string entries take priority over glob entries; an unmatched scope + * (or an absent policy map) defaults to "full". + */ +export function resolveExtractionPolicy( + scope: string, + policy?: Record, +): ExtractionPolicyMode { + if (!policy) return "full"; + if (Object.prototype.hasOwnProperty.call(policy, scope)) { + return policy[scope]; + } + for (const [glob, mode] of Object.entries(policy)) { + if (glob.includes("*") && globToRegExp(glob).test(scope)) { + return mode; + } + } + return "full"; +} + // ============================================================================ // Constants // ============================================================================ @@ -288,6 +333,8 @@ export interface SmartExtractorConfig { workspaceBoundary?: WorkspaceBoundaryConfig; /** Optional admission-control governance layer before downstream dedup/persistence. */ admissionControl?: AdmissionControlConfig; + /** Optional scope-glob -> extraction policy map (Option C). Unmatched scopes default to "full". */ + extractionPolicy?: Record; /** Optional sink for durable reject-audit logging. */ onAdmissionRejected?: (entry: AdmissionRejectionAuditEntry) => Promise | void; /** Optional sink invoked after a memory is successfully created or merged (e.g. markdown mirror). */ @@ -401,15 +448,33 @@ export class SmartExtractor { : [targetScope]; const agentId = options.agentId; + // Option C: scope-glob extraction policy — "none" skips extraction + // entirely, with zero LLM calls, before grounding is ever considered. + const policyMode = resolveExtractionPolicy(targetScope, this.config.extractionPolicy); + if (policyMode === "none") { + this.log( + `memory-pro: smart-extractor: extraction policy "none" for scope ${targetScope}, skipping extraction`, + ); + return stats; + } + // Step 1: LLM extraction - const extraction = await this.extractCandidates(conversationText); + const extraction = await this.extractCandidates(conversationText, policyMode); const candidates = extraction.candidates; if (candidates.length === 0) { this.log("memory-pro: smart-extractor: no memories extracted"); - if (extraction.status === "ok") { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); + } else if (extraction.status === "ok") { + // The model DID emit candidates; validation dropped or demoted them + // all. That is a policy verdict about those candidates, not evidence + // the conversation is noise — learning it as noise would pre-filter + // similar real content away from future extractions. + this.log( + `memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`, + ); } else { this.debugLog( `memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`, @@ -755,6 +820,7 @@ export class SmartExtractor { */ private async extractCandidates( conversationText: string, + policyMode: ExtractionPolicyMode = "full", ): Promise { const maxChars = this.config.extractMaxChars ?? 8000; const truncated = @@ -771,11 +837,13 @@ export class SmartExtractor { const prompt = buildExtractionPrompt(cleaned, user); const result = await this.llm.completeJson<{ + conversation_register?: string; memories: Array<{ category: string; abstract: string; overview: string; content: string; + grounding?: string; }>; }>(prompt, "extract-candidates"); @@ -796,11 +864,27 @@ export class SmartExtractor { `memory-lancedb-pro: smart-extractor: extract-candidates raw memories=${result.memories.length}`, ); + // Batch-level register signal, judged once per extraction. The model + // classifies whole sessions far more reliably than it self-tags single + // items, so the register deterministically overrides per-item grounding + // wobble below. Missing/unrecognized values fail toward scrutiny + // ("mixed"), never toward open. + const rawRegister = + typeof result.conversation_register === "string" + ? result.conversation_register.toLowerCase().trim() + : ""; + const conversationRegister: ConversationRegister = + rawRegister === "real" || rawRegister === "fiction" ? rawRegister : "mixed"; + // Validate and normalize candidates const candidates: CandidateMemory[] = []; let invalidCategoryCount = 0; let shortAbstractCount = 0; let noiseAbstractCount = 0; + let policyDroppedCount = 0; + let constructedDroppedCount = 0; + let fictionRegisterDroppedCount = 0; + let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -838,14 +922,89 @@ export class SmartExtractor { continue; } - candidates.push({ category, abstract, overview, content }); + // Option C: scope policy restricts extraction to episodic-only, + // independent of grounding — checked before the grounding filter below. + if (policyMode === "episodic-only" && category !== "events") { + policyDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping candidate due to episodic-only extraction policy category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + + // Option A / v3: grounding-aware filter. Missing/non-string/unrecognized + // per-item values fail open to "real" so a model that ignores the field + // can't break extraction. Grounding describes the truth-grounding of the + // ASSERTION itself: "real" includes an assertion ABOUT a fiction/game + // session (e.g. that it happened); "constructed" is a claim true only + // WITHIN the fiction. A constructed-tagged candidate is never stored, + // in any category or register — there is no per-extraction cap anymore. + const rawGrounding = + typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; + const grounding: CandidateGrounding = + rawGrounding === "constructed" ? "constructed" : "real"; + if (rawGrounding === "constructed") { + batchHasConstructedTag = true; + } + + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override). + if (conversationRegister === "fiction" && DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + + // Grounding enforcement: a constructed assertion is true only within + // the fiction, never about the real world — never stored, regardless + // of category or register. + if (grounding === "constructed") { + constructedDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + + candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); + } + + // Batch contradiction check: when the batch shows evidence of a + // constructed frame (register not "real" AND at least one item the model + // itself tagged constructed), surviving real-tagged durables in the same + // batch are the classic mislabel shape — fiction items where only the + // grounding tag wobbled. Demote them deterministically, no extra LLM call. + let contradictionDemotedCount = 0; + if (conversationRegister !== "real" && batchHasConstructedTag) { + for (let i = candidates.length - 1; i >= 0; i--) { + const candidate = candidates[i]; + if (DURABLE_CATEGORIES.has(candidate.category)) { + contradictionDemotedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: demoting real-tagged durable from ${conversationRegister}-register batch with constructed siblings (batch contradiction) category=${candidate.category} abstract=${JSON.stringify(candidate.abstract.slice(0, 120))}`, + ); + candidates.splice(i, 1); + } + } + } + + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log( + `memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`, + ); } this.debugLog( - `memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}`, + `memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`, ); - return { status: "ok", candidates }; + return { status: "ok", candidates, rawCandidateCount: result.memories.length }; } // -------------------------------------------------------------------------- @@ -1738,6 +1897,12 @@ export class SmartExtractor { suppressed_until_turn: 0, memory_temporal_type: classifyTemporal(classifyText), valid_until: inferExpiry(classifyText), + // Grounding audit trail: which self-tag and batch register this + // memory was admitted under. Absent on legacy payloads. + ...(candidate.grounding ? { grounding: candidate.grounding } : {}), + ...(candidate.conversationRegister + ? { conversation_register: candidate.conversationRegister } + : {}), ...(admissionAudit ? { admission_audit: JSON.stringify(admissionAudit) } : {}), }, ), diff --git a/test/extraction-grounding-register.test.mjs b/test/extraction-grounding-register.test.mjs new file mode 100644 index 00000000..23bab1da --- /dev/null +++ b/test/extraction-grounding-register.test.mjs @@ -0,0 +1,651 @@ +/** + * Regression tests for grounding-aware extraction (Option A, v3 semantics) + * and the scope-glob extraction-policy knob (Option C). + * + * extractCandidates() tags each raw candidate "grounding": "real" | "constructed". + * v3 grounding is about the truth-grounding of the ASSERTION itself, not which + * register the conversation happened in: "real" includes an assertion ABOUT a + * fiction/game session (e.g. that the session happened); "constructed" is a + * claim true only WITHIN the fiction (game canon, a persona's invented traits). + * A constructed-tagged candidate is NEVER stored, in any category or register — + * there is no per-extraction cap. A genuine first-person aside spoken during + * play is tagged "real" and must survive under its natural category. + * + * Fixtures are entirely synthetic (agent-one/agent-two, invented game + * content) — no real fleet data. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { SmartExtractor, resolveExtractionPolicy } = jiti("../src/smart-extractor.ts"); + +// ============================================================================ +// Helpers +// ============================================================================ + +/** + * One-hot embedder keyed by a text hash. Distinct abstracts land on distinct + * dimensions, so cosine similarity between any two different fixture texts is + * 0 — this keeps Step 1b's batch-internal dedup (threshold 0.85) from + * accidentally collapsing genuinely-different candidates in these tests. + * (A naive char-code-based mock makes short English strings look >0.85 + * "similar" to each other purely from shared letter frequency — verified + * against this file's fixtures before choosing this approach.) + */ +function hashToIndex(text, dims) { + let h = 0; + for (let i = 0; i < text.length; i++) { + h = (h * 31 + text.charCodeAt(i)) >>> 0; + } + return h % dims; +} + +function makeEmbedder(dims = 97) { + const embed = async (text) => { + const v = new Array(dims).fill(0); + v[hashToIndex(text || "", dims)] = 1; + return v; + }; + return { + embed, + async embedBatch(texts) { + return Promise.all((texts || []).map((t) => embed(t))); + }, + }; +} + +/** + * `memories` is the raw memories array the mocked "extract-candidates" LLM + * call returns; `conversationRegister` (optional) is the batch-level + * conversation_register field — omitted to simulate legacy payloads. + */ +function makeLlm(memories, conversationRegister) { + let extractCandidatesCalls = 0; + return { + async completeJson(_prompt, mode) { + if (mode !== "extract-candidates") return null; + extractCandidatesCalls++; + return conversationRegister + ? { conversation_register: conversationRegister, memories } + : { memories }; + }, + get extractCandidatesCalls() { + return extractCandidatesCalls; + }, + }; +} + +function makeStore() { + const bulkStoreCalls = []; + return { + async vectorSearch() { return []; }, + async store(entry) { return entry; }, + async bulkStore(entries) { + bulkStoreCalls.push(entries); + return entries; + }, + async update() {}, + async getById() { return null; }, + get bulkStoreCalls() { return bulkStoreCalls; }, + }; +} + +function makeExtractor(embedder, llm, store, config = {}) { + return new SmartExtractor(store, embedder, llm, { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + log() {}, + debugLog() {}, + ...config, + }); +} + +/** Flatten every category persisted across this extraction's bulkStore call(s), in order. */ +function persistedCategories(store) { + return store.bulkStoreCalls.flat().map((entry) => { + const meta = JSON.parse(entry.metadata || "{}"); + return meta.memory_category; + }); +} + +// ============================================================================ +// Fixtures — synthetic two-agent transcript, invented game content +// ============================================================================ + +const GAME_TRANSCRIPT = [ + "agent-one: let's play a two-round puzzle guessing game, loser buys drinks", + "agent-two: deal, I'll set the rule: the answer is always an even number", + "agent-one: round 1 answer is 42", + "agent-two: round 2 answer is 8, I win the bet", +].join("\n"); + +const GAME_CANDIDATES = [ + { + category: "preferences", + abstract: "House rule: puzzle answers must be even numbers", + overview: "## Rule\n- Even numbers only", + content: "agent-two's puzzle house rule is that answers must always be even numbers.", + grounding: "constructed", // within-the-fiction: true only inside the game's rules + }, + { + category: "cases", + abstract: "Puzzle round 2 answer was 8", + overview: "## Answer\n- 8", + content: "The round 2 puzzle answer was 8, so agent-two won the bet.", + grounding: "constructed", // within-the-fiction: true only inside the game's outcome + }, + { + category: "events", + abstract: "agent-one and agent-two ran a two-round puzzle exercise", + overview: "## What happened\n- Two agents played a puzzle guessing game", + content: "agent-one and agent-two ran a two-round puzzle guessing exercise with an invented house rule and a bet.", + grounding: "real", // about-the-fiction: a true statement that the session happened + }, +]; + +/** A within-the-fiction plot beat — true only inside the story, not a real occurrence. */ +const CONSTRUCTED_PLOT_EVENT = { + category: "events", + abstract: "Admiral Vex's ship was boarded by pirates", + overview: "## In-story plot beat\n- Ship boarded mid-roleplay", + content: "In the roleplay, Admiral Vex's ship was boarded by pirates.", + grounding: "constructed", +}; + +const REAL_ASIDE_CANDIDATE = { + category: "events", + abstract: "operators new laptop arrives Thursday", + overview: "## Real-world aside\n- New laptop arrives Thursday", + content: "In the middle of the puzzle game, the operator mentioned their new laptop arrives Thursday.", + grounding: "real", +}; + +const FACTUAL_CANDIDATES = [ + { + category: "preferences", + abstract: "Python code style: no type hints, concise", + overview: "## Preference\n- No type hints", + content: "User prefers Python code without type hints.", + grounding: "real", + }, + { + category: "profile", + abstract: "User basic info: backend engineer", + overview: "## Background\n- Backend engineer", + content: "User is a backend engineer.", + // no grounding field at all — regression guard for the default extraction path + }, +]; + +// ============================================================================ +// Option A — grounding-aware extraction +// ============================================================================ + +describe("SmartExtractor grounding-aware extraction (Option A, v3)", () => { + it("drops constructed candidates from a game transcript, keeping only the real session-events note", async () => { + const store = makeStore(); + const llm = makeLlm(GAME_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual(categories, ["events"], "only the real (about-the-fiction) session-events note should survive"); + assert.equal(stats.created, 1); + }); + + it("drops a constructed-tagged events candidate too — no cap, unconditional drop regardless of category", async () => { + const store = makeStore(); + const twoConstructedEventsNotes = [ + CONSTRUCTED_PLOT_EVENT, + { + ...CONSTRUCTED_PLOT_EVENT, + abstract: "Admiral Vex also repelled a second boarding attempt", + }, + ]; + const llm = makeLlm(twoConstructedEventsNotes); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.equal(categories.length, 0, "constructed-tagged events candidates are dropped unconditionally in v3 — the old one-per-extraction cap no longer applies"); + assert.equal(stats.created, 0); + }); + + it("keeps a genuine real first-person aside stated during play (false-positive guard)", async () => { + const store = makeStore(); + const llm = makeLlm([...GAME_CANDIDATES, REAL_ASIDE_CANDIDATE]); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const abstracts = store.bulkStoreCalls.flat().map((e) => e.text); + assert.ok( + abstracts.some((a) => a.includes("laptop arrives Thursday")), + "the real aside must survive even though it was stated during a constructed register", + ); + // The real session-events note + the real aside both land as "events". + const categories = persistedCategories(store); + assert.equal(categories.filter((c) => c === "events").length, 2); + assert.equal(categories.length, 2, "constructed non-events candidates must still be dropped"); + }); + + it("invariant: no stored row ever carries grounding 'constructed' — events included, no cap logic remains", async () => { + const store = makeStore(); + const llm = makeLlm( + [GAME_CANDIDATES[0], GAME_CANDIDATES[1], CONSTRUCTED_PLOT_EVENT, GAME_CANDIDATES[2]], + "mixed", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const metas = store.bulkStoreCalls.flat().map((e) => JSON.parse(e.metadata || "{}")); + assert.ok(metas.every((m) => m.grounding !== "constructed"), "no persisted row may carry grounding: constructed"); + assert.deepEqual(persistedCategories(store), ["events"], "only the real session-events note survives; every constructed-tagged candidate is dropped regardless of category"); + assert.equal(stats.created, 1); + }); + + it("leaves a purely factual transcript unchanged (regression guard)", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1"); + + assert.equal(stats.created, 2, "both real/default-real candidates must be extracted, unchanged from current behavior"); + const categories = persistedCategories(store).sort(); + assert.deepEqual(categories, ["preferences", "profile"].sort()); + }); + + it("fails open to 'real' for a malformed/missing grounding field", async () => { + const store = makeStore(); + const llm = makeLlm([ + { category: "preferences", abstract: "Prefers dark mode UI themes", overview: "", content: "User prefers dark mode.", grounding: 12345 }, + { category: "entities", abstract: "Project Foo status: active", overview: "", content: "Project Foo is active.", grounding: "unsure" }, + ]); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some conversation", "s1"); + + assert.equal(stats.created, 2, "non-string/unrecognized grounding values must fail open to 'real' and be kept"); + }); + + it("buildExtractionPrompt documents the v3 grounding contract (structural check)", async () => { + const { buildExtractionPrompt } = jiti("../src/extraction-prompts.ts"); + const prompt = buildExtractionPrompt("some conversation", "test-user"); + + assert.match(prompt, /grounding/i); + assert.match(prompt, /"real"\s*\|\s*"constructed"|real.*constructed/i); + assert.match(prompt, /about-the-fiction is real/i, "the v3 about/within one-line rule must be present"); + assert.match(prompt, /within-the-fiction/i, "the within-the-fiction definition of constructed must be present"); + assert.doesNotMatch(prompt, /at most one/i, "the per-extraction constructed cap must be fully removed from the prompt"); + }); +}); + +// ============================================================================ +// Option C — scope-glob extraction policy knob +// ============================================================================ + +describe("SmartExtractor scope-glob extraction policy (Option C)", () => { + it("resolveExtractionPolicy: exact match wins over glob, unmatched scope defaults to full", () => { + const policy = { "play/*": "none", "play/exact-room": "episodic-only" }; + assert.equal(resolveExtractionPolicy("play/exact-room", policy), "episodic-only"); + assert.equal(resolveExtractionPolicy("play/other-room", policy), "none"); + assert.equal(resolveExtractionPolicy("work/room", policy), "full"); + assert.equal(resolveExtractionPolicy("anything", undefined), "full"); + }); + + it("scope mapped 'none' skips extraction entirely with zero LLM calls", async () => { + const store = makeStore(); + const llm = makeLlm(GAME_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "none" }, + }); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1", { scope: "play/room-1" }); + + assert.equal(llm.extractCandidatesCalls, 0, "no LLM call should be made for a 'none'-policy scope"); + assert.deepEqual(stats, { created: 0, merged: 0, skipped: 0, boundarySkipped: 0 }); + }); + + it("scope mapped 'episodic-only' keeps only events-class candidates, independent of grounding", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES.concat([{ + category: "events", + abstract: "User shipped the v2 release", + overview: "", + content: "User shipped the v2 release.", + grounding: "real", + }])); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "episodic-only" }, + }); + + await extractor.extractAndPersist("some conversation", "s1", { scope: "play/room-1" }); + + const categories = persistedCategories(store); + assert.deepEqual(categories, ["events"], "only events-class candidates should survive under episodic-only policy"); + }); + + it("unmatched scope leaves extraction behavior unchanged ('full')", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "none" }, + }); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1", { scope: "work/room" }); + + assert.equal(llm.extractCandidatesCalls, 1); + assert.equal(stats.created, 2); + }); +}); + +// ============================================================================ +// Grounding v2 — batch register signal, contradiction check, propagation, +// grounding-aware admission +// ============================================================================ + +/** Fiction-frame durables the extractor mislabeled as "real" (Mode A shape). */ +const MISLABELED_FICTION_CANDIDATES = [ + { + category: "profile", + abstract: "User lives in Moon Base 9", + overview: "## Background\n- Residence: Moon Base 9", + content: "User lives in Moon Base 9.", + grounding: "real", // wrong: within-the-fiction canon should be "constructed" + }, + { + category: "preferences", + abstract: "Favorite drink is nebula tea", + overview: "## Preference\n- Drink: nebula tea", + content: "User's favorite drink is nebula tea.", + grounding: "real", // wrong: within-the-fiction canon should be "constructed" + }, + { + category: "events", + abstract: "User and assistant played one round of a space roleplay game", + overview: "## What happened\n- One roleplay round", + content: "User and assistant played one round of a space roleplay game this session.", + grounding: "real", // correct under v3: about-the-fiction, a true statement that the session happened + }, +]; + +describe("SmartExtractor batch register signal (grounding v2)", () => { + it("register 'fiction' drops ALL durable candidates even when their per-item tags say 'real'", async () => { + const store = makeStore(); + const llm = makeLlm(MISLABELED_FICTION_CANDIDATES, "fiction"); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual( + categories, + ["events"], + "mislabeled real-tagged durables must not survive a fiction-register batch", + ); + assert.equal(stats.created, 1); + }); + + it("register 'fiction' no longer caps events notes — real session-events notes all survive (v3: cap removed)", async () => { + const store = makeStore(); + const llm = makeLlm( + [ + MISLABELED_FICTION_CANDIDATES[2], + { + ...MISLABELED_FICTION_CANDIDATES[2], + abstract: "User and assistant also played a bonus round", + }, + ], + "fiction", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + assert.equal(persistedCategories(store).length, 2, "both real session-events notes survive; the v2 one-per-extraction cap no longer applies"); + }); + + it("register 'mixed' with a constructed sibling demotes real-tagged durables (batch contradiction check) and drops the constructed sibling itself", async () => { + const store = makeStore(); + const llm = makeLlm( + [ + MISLABELED_FICTION_CANDIDATES[0], // profile tagged real (mislabeled) + CONSTRUCTED_PLOT_EVENT, // events tagged constructed — the contradiction evidence, and itself dropped unconditionally + REAL_ASIDE_CANDIDATE, // real events aside must still survive + ], + "mixed", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.ok(!categories.includes("profile"), "real-tagged durable must be demoted in a mixed-register batch with constructed siblings"); + assert.deepEqual(categories, ["events"], "only the real aside survives: the constructed plot event is dropped unconditionally (v3), profile demoted by contradiction check"); + }); + + it("register 'real' fully trusts per-item tags for durables, but still drops a constructed-tagged item regardless of register", async () => { + const store = makeStore(); + const llm = makeLlm( + [FACTUAL_CANDIDATES[0], CONSTRUCTED_PLOT_EVENT], + "real", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist("mostly factual conversation with a brief game", "s1"); + + const categories = persistedCategories(store); + assert.deepEqual(categories, ["preferences"], "durable real-tagged preference survives; the constructed-tagged item is dropped even though the register is 'real'"); + }); + + it("missing register (legacy payload) with no constructed tags behaves exactly as before", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); // no conversation_register field + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1"); + + assert.equal(stats.created, 2, "legacy payloads (no register, no constructed tags) must be unaffected"); + }); + + it("missing register (legacy payload) WITH a constructed sibling still arms the batch-wide durable wipe", async () => { + // Complements the test above: a missing/malformed register defaults to "mixed" (not + // "real"), and — unlike the no-constructed-tags case — a genuinely constructed sibling + // in the same batch is enough to arm the same wipe a "fiction"/"mixed" register would. + // This is the compound path a malformed extraction response could realistically hit + // (a legacy or dropped conversation_register field alongside a real constructed note), + // which the existing per-register tests don't individually cover. + // + // MISLABELED_FICTION_CANDIDATES[2] no longer fits this fixture's role under v3: that + // abstract ("the session happened") is now correctly tagged "real" (about-the-fiction), + // so it can no longer arm the wipe. Seed an explicit constructed sibling instead + // (CONSTRUCTED_PLOT_EVENT, a within-the-fiction plot beat) alongside a genuine real + // events aside so something still survives to prove the durable-only wipe, not a + // blanket drop of everything. + const store = makeStore(); + const llm = makeLlm([ + FACTUAL_CANDIDATES[0], // preferences, durable — must be wiped by the constructed sibling below + CONSTRUCTED_PLOT_EVENT, // within-the-fiction: arms the wipe, and is itself dropped unconditionally + REAL_ASIDE_CANDIDATE, // real events aside must still survive + ]); // no conversation_register field + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual( + categories, + ["events"], + "a constructed sibling arms the wipe even when conversation_register is missing entirely", + ); + assert.equal(stats.created, 1); + }); + + it("propagates grounding and conversation_register into stored metadata", async () => { + const store = makeStore(); + const llm = makeLlm( + [FACTUAL_CANDIDATES[0], MISLABELED_FICTION_CANDIDATES[2]], + "real", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist("mostly factual conversation with a brief game", "s1"); + + const metas = store.bulkStoreCalls.flat().map((e) => JSON.parse(e.metadata || "{}")); + const prefMeta = metas.find((m) => m.memory_category === "preferences"); + const eventsMeta = metas.find((m) => m.memory_category === "events"); + assert.equal(prefMeta.grounding, "real"); + assert.equal(prefMeta.conversation_register, "real"); + assert.equal(eventsMeta.grounding, "real"); + assert.equal(eventsMeta.conversation_register, "real"); + }); + + it("buildExtractionPrompt documents the batch register contract (structural check)", () => { + const { buildExtractionPrompt } = jiti("../src/extraction-prompts.ts"); + const prompt = buildExtractionPrompt("some conversation", "test-user"); + + assert.match(prompt, /conversation_register/); + assert.match(prompt, /"real\|mixed\|fiction"/); + assert.match(prompt, /self-consistency/i, "the batch self-consistency instruction must be present"); + assert.doesNotMatch(prompt, /storage rule applied after tagging/i, "the deleted per-extraction cap language must not remain in the prompt"); + }); +}); + +describe("AdmissionController grounding awareness (grounding v2)", () => { + const { AdmissionController, ADMISSION_CONTROL_PRESETS, scoreGroundedTypePrior } = jiti("../src/admission-control.ts"); + const balanced = ADMISSION_CONTROL_PRESETS.balanced; + + function makeAdmissionLlm() { + let utilityCalls = 0; + return { + async completeJson(_prompt, mode) { + if (mode === "admission-utility") { + utilityCalls++; + return { utility: 0.9, reason: "mock" }; + } + return null; + }, + get utilityCalls() { + return utilityCalls; + }, + }; + } + + const admissionStore = { async vectorSearch() { return []; } }; + + it("short-circuits constructed durable candidates to reject with zero LLM calls", async () => { + const llm = makeAdmissionLlm(); + const controller = new AdmissionController(admissionStore, llm, balanced); + + const evaluation = await controller.evaluate({ + candidate: { + category: "preferences", + abstract: "House rule: puzzle answers must be even numbers", + overview: "## Rule\n- Even numbers only", + content: "The puzzle house rule is that answers must always be even numbers.", + grounding: "constructed", + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(evaluation.decision, "reject"); + assert.equal(llm.utilityCalls, 0, "the short-circuit must not spend an LLM call"); + assert.match(evaluation.audit.reason, /constructed-grounding/); + assert.equal(evaluation.audit.grounding, "constructed"); + assert.equal(evaluation.audit.conversation_register, "fiction"); + }); + + it("v3: short-circuits constructed candidates of ANY category to reject with zero LLM calls (total enforcement)", async () => { + const llm = makeAdmissionLlm(); + const controller = new AdmissionController(admissionStore, llm, balanced); + + const evaluation = await controller.evaluate({ + candidate: { + category: "events", + abstract: "Admiral Vex's ship was boarded by pirates", + overview: "## In-story plot beat\n- Ship boarded mid-roleplay", + content: "In the roleplay, Admiral Vex's ship was boarded by pirates.", + grounding: "constructed", + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(evaluation.decision, "reject"); + assert.equal(llm.utilityCalls, 0, "v3 total enforcement must short-circuit events too, not just durable categories"); + assert.match(evaluation.audit.reason, /constructed-grounding/); + assert.equal(evaluation.audit.grounding, "constructed"); + }); + + it("scoreGroundedTypePrior caps durable priors at the events prior for fiction-register candidates", () => { + const priors = balanced.typePriors; + const fictionProfile = { + category: "profile", + abstract: "x", + overview: "", + content: "", + grounding: "real", + conversationRegister: "fiction", + }; + const realProfile = { ...fictionProfile, conversationRegister: "real" }; + + assert.equal(scoreGroundedTypePrior(fictionProfile, priors), priors.events, "fiction register must cap the profile prior at the events prior"); + assert.equal(scoreGroundedTypePrior(realProfile, priors), priors.profile, "real register keeps the raw prior"); + }); + + it("buildUtilityPrompt interpolates grounding and names all six registers (structural check)", async () => { + const llm = { + prompts: [], + async completeJson(prompt, mode) { + if (mode === "admission-utility") { + this.prompts.push(prompt); + return { utility: 0.5, reason: "mock" }; + } + return null; + }, + }; + const controller = new AdmissionController(admissionStore, llm, balanced); + + await controller.evaluate({ + candidate: { + category: "events", + abstract: "User and assistant played one round of a space roleplay game", + overview: "", + content: "One roleplay round happened.", + grounding: "real", // v3: real candidates reach normal scoring; constructed short-circuits before this prompt is ever built + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(llm.prompts.length, 1); + const prompt = llm.prompts[0]; + // The judge does not see a per-candidate "Grounding:"/"Conversation + // register:" field (the candidate block is transcript-free and shared + // with every other admission path) -- grounding enforcement rides the + // deterministic constructed short-circuit (this candidate is "real", so + // it reaches normal scoring) plus this prose rule paragraph, both + // carried in the system prompt. + assert.match(prompt, /roleplay|fiction|hypothetical|simulation/i); + for (const register of ["profile", "preferences", "entities", "events", "cases", "patterns"]) { + assert.match(prompt, new RegExp(register), `utility prompt must name the ${register} register`); + } + assert.match(prompt, /roleplay/i, "utility prompt must carry the fiction guidance"); + }); +}); diff --git a/test/smart-extractor-noise-gating.test.mjs b/test/smart-extractor-noise-gating.test.mjs index 2382fca9..0a181624 100644 --- a/test/smart-extractor-noise-gating.test.mjs +++ b/test/smart-extractor-noise-gating.test.mjs @@ -4,10 +4,12 @@ * extractCandidates() can come back with zero candidates for four different * reasons: the LLM/gateway call failed outright, the response had an * unexpected shape, the LLM genuinely returned an empty memories list, or - * every parsed candidate was dropped by local validation. Only the latter - * two are a real "nothing to remember" signal — the first two are failures - * that must NOT be fed into the noise-prototype bank, or gateway outages - * silently poison future extraction/recall. + * every parsed candidate was dropped/demoted by local validation. ONLY the + * genuinely-empty list is a real "nothing to remember" signal. Failures must + * not train the bank (gateway outages would poison it), and a + * validation-emptied batch must not either: the model DID find candidates + * there, so teaching the bank "this conversation is noise" would pre-filter + * similar real content away from future extractions. */ import { describe, it } from "node:test"; @@ -42,6 +44,7 @@ function makeLlm(behavior) { if (behavior === "malformed_non_array") return { memories: "not-an-array" }; if (behavior === "empty") return { memories: [] }; if (Array.isArray(behavior)) return { memories: behavior }; + if (behavior && typeof behavior === "object" && behavior.raw) return behavior.raw; throw new Error(`unsupported test behavior: ${behavior}`); }, }; @@ -130,16 +133,69 @@ describe("SmartExtractor noise-bank learning gate", () => { assert.equal(noiseBank.learnCalls.length, 1, "a genuine empty list is a real noise signal"); }); - it("learns noise when every parsed candidate is dropped by local validation", async () => { + it("does NOT learn noise when validation drops every parsed candidate (policy verdict, not a noise signal)", async () => { const noiseBank = makeNoiseBank(); + const logs = []; // "hi" is a valid category but fails the length>=5 abstract check, so the - // parsed response is well-formed yet every candidate is filtered out. + // parsed response is well-formed yet every candidate is filtered out. The + // model DID emit a candidate, so the conversation must not train the bank. const llm = makeLlm([{ category: "preferences", abstract: "hi", overview: "", content: "" }]); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + assert.equal(noiseBank.learnCalls.length, 0, "a validation-emptied batch must not train the noise bank"); + assert.ok( + logs.some((msg) => msg.includes("skipping noise-bank learning (validation emptied the batch")), + "the skip must be visible at the standard log level", + ); + }); + + it("does NOT learn noise when the batch contradiction check demotes every candidate", async () => { + const noiseBank = makeNoiseBank(); + // Mixed-register batch with a constructed sibling: the real-tagged durable + // is demoted by the batch contradiction check, the constructed one is + // dropped by grounding enforcement. Raw output had 2 candidates, so the + // conversation (which contains real facts the policy demoted) must not be + // fed to the noise bank as a noise exemplar. + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank }); await extractor.extractAndPersist("some conversation text", "s1"); await flushMicrotasks(); - assert.equal(noiseBank.learnCalls.length, 1, "a validly-parsed-but-empty-after-filtering result is still a real noise signal"); + assert.equal(noiseBank.learnCalls.length, 0, "a demotion-emptied batch must not train the noise bank"); + }); + + it("logs the batch contradiction demotion at the standard log level, with count and register", async () => { + const noiseBank = makeNoiseBank(); + const logs = []; + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + const demotionLine = logs.find((msg) => msg.includes("batch contradiction demoted")); + assert.ok(demotionLine, "a fully-demoted batch must be distinguishable from 'model found nothing' without debug logging"); + assert.ok(demotionLine.includes("1"), "the demotion line must carry the demoted count"); + assert.ok(demotionLine.includes("mixed"), "the demotion line must carry the batch register"); }); });