From 18e36ca404a9a2cffc8b915e21eec24b13f909f3 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 11 Jul 2026 21:13:17 +0300 Subject: [PATCH 01/11] fix(extraction-prompts): tag extraction candidates with conversational grounding Extraction has no notion of register: an invented rule stated inside a game is textually indistinguishable from a durable user preference, so the admission gate (which scores shape, not reality) happily promotes in-fiction assertions to profile/preference facts. Add a "grounding" field ("real" | "constructed") to the extraction output schema so the extractor can flag content that only holds inside an in-conversation constructed context (games, roleplay, drafted fiction, hypotheticals, sample data). A real aside stated in passing during play is still "real" and keeps its natural category. This is prompt-only; the enforcement post-filter lands separately. --- src/extraction-prompts.ts | 41 ++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/extraction-prompts.ts b/src/extraction-prompts.ts index 93a787f4c..eeebbd23c 100644 --- a/src/extraction-prompts.ts +++ b/src/extraction-prompts.ts @@ -70,6 +70,20 @@ ${conversationText} - "Encountered problem A, used solution B" -> cases (not events) - "General process for handling certain problems" -> patterns (not cases) +# Conversational Grounding + +Every memory must also be tagged with how grounded it is in reality: + +| grounding | Meaning | +|-----------|---------| +| "real" | An assertion about the actual user, the real world, or this real working session — including a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | + +Rules: +- When the surrounding register is a game, roleplay, fiction, hypothetical, or sample-data exercise, you may record AT MOST ONE session-scoped "events" memory noting that the real participants did the activity (e.g. "agent-one and agent-two ran a puzzle exercise"). Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. +- A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. +- If you are unsure, default to "real" — under-tagging as constructed risks losing a genuine fact. + # Three-Level Structure Each memory contains three levels: @@ -90,7 +104,8 @@ Each memory contains three levels: "category": "profile", "abstract": "User basic info: AI development engineer, 3 years LLM experience", "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience." + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" } \`\`\` @@ -100,7 +115,8 @@ Each memory contains three levels: "category": "preferences", "abstract": "Python code style: No type hints, concise and direct", "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments." + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" } \`\`\` @@ -110,9 +126,22 @@ Each memory contains three levels: "category": "cases", "abstract": "LanceDB BigInt numeric handling issue", "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations." + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" +} +\`\`\` + +## constructed register (game / roleplay / hypothetical) +\`\`\`json +{ + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "constructed" } \`\`\` +Note: the invented house rule and the bet itself are NOT extracted as separate preferences/cases — only this one episodic note survives. # Output Format @@ -123,7 +152,8 @@ Return JSON: "category": "profile|preferences|entities|events|cases|patterns", "abstract": "One-line index", "overview": "Structured Markdown summary", - "content": "Full narrative" + "content": "Full narrative", + "grounding": "real|constructed" } ] } @@ -133,7 +163,8 @@ Notes: - Only extract truly valuable personalized information - If nothing worth recording, return {"memories": []} - Maximum 5 memories per extraction -- Preferences should be aggregated by topic`; +- Preferences should be aggregated by topic +- Tag every memory's "grounding" field per the Conversational Grounding rules above`; } export function buildDedupPrompt( From 05811fb2318753baf2073b41e0abb7b1ac178ffe Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 11 Jul 2026 21:13:25 +0300 Subject: [PATCH 02/11] fix(smart-extractor): drop constructed-grounding candidates; add scope extraction-policy knob Two independent, deterministic filters in extractCandidates()'s validation loop: - Grounding post-filter (companion to the prompt change): a candidate tagged grounding:"constructed" is dropped unless its category is "events", and at most one constructed "events" note survives per extraction. Missing/non-string/unrecognized grounding values fail open to "real" so a model that ignores the field cannot regress extraction. Real-grounded candidates are never affected by this filter, regardless of category. - Scope-glob extraction-policy knob (config only, no host plumbing required): SmartExtractorConfig.extractionPolicy maps a scope or scope-glob to "full" (default, unchanged) | "episodic-only" (only "events"-class candidates survive, independent of grounding) | "none" (extraction skipped entirely, zero LLM calls). Resolved via the new exported resolveExtractionPolicy(); exact-string entries take priority over glob entries. Neither filter touches admission control or speaker attribution. --- src/smart-extractor.ts | 98 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index b74ac8183..ced1975fb 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -234,6 +234,45 @@ export function stripEnvelopeMetadata(text: string): string { return cleaned.trim(); } +// ============================================================================ +// Extraction Policy (Option C — scope-glob knob) +// ============================================================================ + +/** + * Operator-facing extraction policy for a scope: + * - "full" (default): today's behavior, unchanged. + * - "episodic-only": only "events"-class candidates are kept, regardless of + * grounding — a blunt backstop for scopes known to be pure play/roleplay. + * - "none": extraction is skipped entirely, with zero LLM calls. + */ +export type ExtractionPolicyMode = "full" | "episodic-only" | "none"; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + return new RegExp(`^${escaped}$`); +} + +/** + * Resolve the extraction policy for a scope against a scope-glob -> mode map. + * Exact-string entries take priority over glob entries; an unmatched scope + * (or an absent policy map) defaults to "full". + */ +export function resolveExtractionPolicy( + scope: string, + policy?: Record, +): ExtractionPolicyMode { + if (!policy) return "full"; + if (Object.prototype.hasOwnProperty.call(policy, scope)) { + return policy[scope]; + } + for (const [glob, mode] of Object.entries(policy)) { + if (glob.includes("*") && globToRegExp(glob).test(scope)) { + return mode; + } + } + return "full"; +} + // ============================================================================ // Constants // ============================================================================ @@ -288,6 +327,8 @@ export interface SmartExtractorConfig { workspaceBoundary?: WorkspaceBoundaryConfig; /** Optional admission-control governance layer before downstream dedup/persistence. */ admissionControl?: AdmissionControlConfig; + /** Optional scope-glob -> extraction policy map (Option C). Unmatched scopes default to "full". */ + extractionPolicy?: Record; /** Optional sink for durable reject-audit logging. */ onAdmissionRejected?: (entry: AdmissionRejectionAuditEntry) => Promise | void; /** Optional sink invoked after a memory is successfully created or merged (e.g. markdown mirror). */ @@ -401,8 +442,18 @@ export class SmartExtractor { : [targetScope]; const agentId = options.agentId; + // Option C: scope-glob extraction policy — "none" skips extraction + // entirely, with zero LLM calls, before grounding is ever considered. + const policyMode = resolveExtractionPolicy(targetScope, this.config.extractionPolicy); + if (policyMode === "none") { + this.log( + `memory-pro: smart-extractor: extraction policy "none" for scope ${targetScope}, skipping extraction`, + ); + return stats; + } + // Step 1: LLM extraction - const extraction = await this.extractCandidates(conversationText); + const extraction = await this.extractCandidates(conversationText, policyMode); const candidates = extraction.candidates; if (candidates.length === 0) { @@ -755,6 +806,7 @@ export class SmartExtractor { */ private async extractCandidates( conversationText: string, + policyMode: ExtractionPolicyMode = "full", ): Promise { const maxChars = this.config.extractMaxChars ?? 8000; const truncated = @@ -776,6 +828,7 @@ export class SmartExtractor { abstract: string; overview: string; content: string; + grounding?: string; }>; }>(prompt, "extract-candidates"); @@ -801,6 +854,9 @@ export class SmartExtractor { let invalidCategoryCount = 0; let shortAbstractCount = 0; let noiseAbstractCount = 0; + let policyDroppedCount = 0; + let constructedDroppedCount = 0; + let constructedEpisodicUsed = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -838,11 +894,49 @@ export class SmartExtractor { continue; } + // Option C: scope policy restricts extraction to episodic-only, + // independent of grounding — checked before the grounding filter below. + if (policyMode === "episodic-only" && category !== "events") { + policyDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping candidate due to episodic-only extraction policy category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + + // Option A: grounding-aware register filter. Missing/non-string/unrecognized + // values fail open to "real" so a model that ignores the field can't break + // extraction. Constructed content (games, roleplay, fiction, hypotheticals, + // sample data) never becomes a durable profile/preferences/entities/cases/ + // patterns memory; at most one constructed "events" note per extraction + // records that the real participants did the activity. + const rawGrounding = + typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; + const grounding: "real" | "constructed" = + rawGrounding === "constructed" ? "constructed" : "real"; + if (grounding === "constructed") { + if (category !== "events") { + constructedDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate outside events category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + if (constructedEpisodicUsed) { + constructedDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping extra constructed-grounding events candidate (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + constructedEpisodicUsed = true; + } + candidates.push({ category, abstract, overview, content }); } this.debugLog( - `memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}`, + `memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}`, ); return { status: "ok", candidates }; From 1ea6146eacbfce982dfb92e089aae0b0295cf723 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 11 Jul 2026 21:13:36 +0300 Subject: [PATCH 03/11] test: add grounding-aware extraction and scope-policy regression suite Covers: constructed game content dropped from profile/preferences/ entities/cases; at most one constructed episodic events note per extraction; a genuine real first-person aside stated during play survives (false-positive guard); a purely factual transcript is unchanged; malformed/missing grounding fails open to "real"; the prompt documents the grounding contract; and the scope-glob policy resolver plus its "none"/"episodic-only"/unmatched behavior through extractAndPersist. Fixtures are synthetic (agent-one/agent-two). --- test/extraction-grounding-register.test.mjs | 314 ++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 test/extraction-grounding-register.test.mjs diff --git a/test/extraction-grounding-register.test.mjs b/test/extraction-grounding-register.test.mjs new file mode 100644 index 000000000..13e0a47e2 --- /dev/null +++ b/test/extraction-grounding-register.test.mjs @@ -0,0 +1,314 @@ +/** + * Regression tests for grounding-aware extraction (Option A) and the + * scope-glob extraction-policy knob (Option C). + * + * extractCandidates() tags each raw candidate "grounding": "real" | "constructed". + * A deterministic post-filter drops constructed candidates typed as + * profile/preferences/entities/cases/patterns, and allows at most one + * session-scoped "events" note per extraction for a constructed register + * (game/roleplay/fiction/hypothetical). A genuine first-person aside spoken + * during play is tagged "real" and must survive under its natural category. + * + * Fixtures are entirely synthetic (agent-one/agent-two, invented game + * content) — no real fleet data. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { SmartExtractor, resolveExtractionPolicy } = jiti("../src/smart-extractor.ts"); + +// ============================================================================ +// Helpers +// ============================================================================ + +/** + * One-hot embedder keyed by a text hash. Distinct abstracts land on distinct + * dimensions, so cosine similarity between any two different fixture texts is + * 0 — this keeps Step 1b's batch-internal dedup (threshold 0.85) from + * accidentally collapsing genuinely-different candidates in these tests. + * (A naive char-code-based mock makes short English strings look >0.85 + * "similar" to each other purely from shared letter frequency — verified + * against this file's fixtures before choosing this approach.) + */ +function hashToIndex(text, dims) { + let h = 0; + for (let i = 0; i < text.length; i++) { + h = (h * 31 + text.charCodeAt(i)) >>> 0; + } + return h % dims; +} + +function makeEmbedder(dims = 97) { + const embed = async (text) => { + const v = new Array(dims).fill(0); + v[hashToIndex(text || "", dims)] = 1; + return v; + }; + return { + embed, + async embedBatch(texts) { + return Promise.all((texts || []).map((t) => embed(t))); + }, + }; +} + +/** `memories` is the raw memories array the mocked "extract-candidates" LLM call returns. */ +function makeLlm(memories) { + let extractCandidatesCalls = 0; + return { + async completeJson(_prompt, mode) { + if (mode !== "extract-candidates") return null; + extractCandidatesCalls++; + return { memories }; + }, + get extractCandidatesCalls() { + return extractCandidatesCalls; + }, + }; +} + +function makeStore() { + const bulkStoreCalls = []; + return { + async vectorSearch() { return []; }, + async store(entry) { return entry; }, + async bulkStore(entries) { + bulkStoreCalls.push(entries); + return entries; + }, + async update() {}, + async getById() { return null; }, + get bulkStoreCalls() { return bulkStoreCalls; }, + }; +} + +function makeExtractor(embedder, llm, store, config = {}) { + return new SmartExtractor(store, embedder, llm, { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + log() {}, + debugLog() {}, + ...config, + }); +} + +/** Flatten every category persisted across this extraction's bulkStore call(s), in order. */ +function persistedCategories(store) { + return store.bulkStoreCalls.flat().map((entry) => { + const meta = JSON.parse(entry.metadata || "{}"); + return meta.memory_category; + }); +} + +// ============================================================================ +// Fixtures — synthetic two-agent transcript, invented game content +// ============================================================================ + +const GAME_TRANSCRIPT = [ + "agent-one: let's play a two-round puzzle guessing game, loser buys drinks", + "agent-two: deal, I'll set the rule: the answer is always an even number", + "agent-one: round 1 answer is 42", + "agent-two: round 2 answer is 8, I win the bet", +].join("\n"); + +const GAME_CANDIDATES = [ + { + category: "preferences", + abstract: "House rule: puzzle answers must be even numbers", + overview: "## Rule\n- Even numbers only", + content: "agent-two's puzzle house rule is that answers must always be even numbers.", + grounding: "constructed", + }, + { + category: "cases", + abstract: "Puzzle round 2 answer was 8", + overview: "## Answer\n- 8", + content: "The round 2 puzzle answer was 8, so agent-two won the bet.", + grounding: "constructed", + }, + { + category: "events", + abstract: "agent-one and agent-two ran a two-round puzzle exercise", + overview: "## What happened\n- Two agents played a puzzle guessing game", + content: "agent-one and agent-two ran a two-round puzzle guessing exercise with an invented house rule and a bet.", + grounding: "constructed", + }, +]; + +const REAL_ASIDE_CANDIDATE = { + category: "events", + abstract: "operators new laptop arrives Thursday", + overview: "## Real-world aside\n- New laptop arrives Thursday", + content: "In the middle of the puzzle game, the operator mentioned their new laptop arrives Thursday.", + grounding: "real", +}; + +const FACTUAL_CANDIDATES = [ + { + category: "preferences", + abstract: "Python code style: no type hints, concise", + overview: "## Preference\n- No type hints", + content: "User prefers Python code without type hints.", + grounding: "real", + }, + { + category: "profile", + abstract: "User basic info: backend engineer", + overview: "## Background\n- Backend engineer", + content: "User is a backend engineer.", + // no grounding field at all — regression guard for the default extraction path + }, +]; + +// ============================================================================ +// Option A — grounding-aware extraction +// ============================================================================ + +describe("SmartExtractor grounding-aware extraction (Option A)", () => { + it("drops constructed non-events candidates from a game transcript, keeping only the episodic events note", async () => { + const store = makeStore(); + const llm = makeLlm(GAME_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual(categories, ["events"], "only the single constructed-but-episodic events note should survive"); + assert.equal(stats.created, 1); + }); + + it("caps constructed events notes at one per extraction", async () => { + const store = makeStore(); + const twoEventsNotes = [ + GAME_CANDIDATES[2], + { + ...GAME_CANDIDATES[2], + abstract: "agent-one and agent-two also played a second puzzle round", + }, + ]; + const llm = makeLlm(twoEventsNotes); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.equal(categories.length, 1, "only the first constructed events note should survive the cap"); + assert.equal(stats.created, 1); + }); + + it("keeps a genuine real first-person aside stated during play (false-positive guard)", async () => { + const store = makeStore(); + const llm = makeLlm([...GAME_CANDIDATES, REAL_ASIDE_CANDIDATE]); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const abstracts = store.bulkStoreCalls.flat().map((e) => e.text); + assert.ok( + abstracts.some((a) => a.includes("laptop arrives Thursday")), + "the real aside must survive even though it was stated during a constructed register", + ); + // The constructed episodic note + the real aside both land as "events". + const categories = persistedCategories(store); + assert.equal(categories.filter((c) => c === "events").length, 2); + assert.equal(categories.length, 2, "constructed non-events candidates must still be dropped"); + }); + + it("leaves a purely factual transcript unchanged (regression guard)", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1"); + + assert.equal(stats.created, 2, "both real/default-real candidates must be extracted, unchanged from current behavior"); + const categories = persistedCategories(store).sort(); + assert.deepEqual(categories, ["preferences", "profile"].sort()); + }); + + it("fails open to 'real' for a malformed/missing grounding field", async () => { + const store = makeStore(); + const llm = makeLlm([ + { category: "preferences", abstract: "Prefers dark mode UI themes", overview: "", content: "User prefers dark mode.", grounding: 12345 }, + { category: "entities", abstract: "Project Foo status: active", overview: "", content: "Project Foo is active.", grounding: "unsure" }, + ]); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some conversation", "s1"); + + assert.equal(stats.created, 2, "non-string/unrecognized grounding values must fail open to 'real' and be kept"); + }); + + it("buildExtractionPrompt documents the grounding contract (structural check)", async () => { + const { buildExtractionPrompt } = jiti("../src/extraction-prompts.ts"); + const prompt = buildExtractionPrompt("some conversation", "test-user"); + + assert.match(prompt, /grounding/i); + assert.match(prompt, /"real"\s*\|\s*"constructed"|real.*constructed/i); + assert.match(prompt, /at most one/i); + }); +}); + +// ============================================================================ +// Option C — scope-glob extraction policy knob +// ============================================================================ + +describe("SmartExtractor scope-glob extraction policy (Option C)", () => { + it("resolveExtractionPolicy: exact match wins over glob, unmatched scope defaults to full", () => { + const policy = { "play/*": "none", "play/exact-room": "episodic-only" }; + assert.equal(resolveExtractionPolicy("play/exact-room", policy), "episodic-only"); + assert.equal(resolveExtractionPolicy("play/other-room", policy), "none"); + assert.equal(resolveExtractionPolicy("work/room", policy), "full"); + assert.equal(resolveExtractionPolicy("anything", undefined), "full"); + }); + + it("scope mapped 'none' skips extraction entirely with zero LLM calls", async () => { + const store = makeStore(); + const llm = makeLlm(GAME_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "none" }, + }); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1", { scope: "play/room-1" }); + + assert.equal(llm.extractCandidatesCalls, 0, "no LLM call should be made for a 'none'-policy scope"); + assert.deepEqual(stats, { created: 0, merged: 0, skipped: 0, boundarySkipped: 0 }); + }); + + it("scope mapped 'episodic-only' keeps only events-class candidates, independent of grounding", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES.concat([{ + category: "events", + abstract: "User shipped the v2 release", + overview: "", + content: "User shipped the v2 release.", + grounding: "real", + }])); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "episodic-only" }, + }); + + await extractor.extractAndPersist("some conversation", "s1", { scope: "play/room-1" }); + + const categories = persistedCategories(store); + assert.deepEqual(categories, ["events"], "only events-class candidates should survive under episodic-only policy"); + }); + + it("unmatched scope leaves extraction behavior unchanged ('full')", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); + const extractor = makeExtractor(makeEmbedder(), llm, store, { + extractionPolicy: { "play/*": "none" }, + }); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1", { scope: "work/room" }); + + assert.equal(llm.extractCandidatesCalls, 1); + assert.equal(stats.created, 2); + }); +}); From 74e8c0c044b22833cf275a69071b161111950213 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 11 Jul 2026 21:14:24 +0300 Subject: [PATCH 04/11] chore: wire extraction-grounding-register suite into CI test chain Adds the new suite to package.json's test script and to scripts/ci-test-manifest.mjs under core-regression, alongside the other smart-extractor regression suites. --- package.json | 2 +- scripts/ci-test-manifest.mjs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 8260b2e86..0baf407e7 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/extraction-grounding-register.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", "test:llm-clients-and-auth": "node scripts/run-ci-tests.mjs --group llm-clients-and-auth", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index dce4388f0..142b655e2 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -37,6 +37,7 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-noise-gating.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/extraction-grounding-register.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-capability-runtime.test.mjs" }, { group: "core-regression", runner: "node", file: "test/dreaming-engine.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-governance-tools.test.mjs", args: ["--test"] }, From a98b11c2f68049deb69e4cd6615a61dc151e54cd Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 11 Jul 2026 21:19:01 +0300 Subject: [PATCH 05/11] chore: rebuild dist for grounding-aware extraction and extraction-policy knob Compiled output for the extraction-prompts.ts grounding tag and the smart-extractor.ts post-filter + scope-glob extraction-policy knob. --- dist/src/extraction-prompts.js | 41 ++++++++++++++++++--- dist/src/smart-extractor.js | 66 ++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/dist/src/extraction-prompts.js b/dist/src/extraction-prompts.js index b74c9b3d9..008f0a2da 100644 --- a/dist/src/extraction-prompts.js +++ b/dist/src/extraction-prompts.js @@ -66,6 +66,20 @@ ${conversationText} - "Encountered problem A, used solution B" -> cases (not events) - "General process for handling certain problems" -> patterns (not cases) +# Conversational Grounding + +Every memory must also be tagged with how grounded it is in reality: + +| grounding | Meaning | +|-----------|---------| +| "real" | An assertion about the actual user, the real world, or this real working session — including a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | + +Rules: +- When the surrounding register is a game, roleplay, fiction, hypothetical, or sample-data exercise, you may record AT MOST ONE session-scoped "events" memory noting that the real participants did the activity (e.g. "agent-one and agent-two ran a puzzle exercise"). Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. +- A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. +- If you are unsure, default to "real" — under-tagging as constructed risks losing a genuine fact. + # Three-Level Structure Each memory contains three levels: @@ -86,7 +100,8 @@ Each memory contains three levels: "category": "profile", "abstract": "User basic info: AI development engineer, 3 years LLM experience", "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience." + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" } \`\`\` @@ -96,7 +111,8 @@ Each memory contains three levels: "category": "preferences", "abstract": "Python code style: No type hints, concise and direct", "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments." + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" } \`\`\` @@ -106,9 +122,22 @@ Each memory contains three levels: "category": "cases", "abstract": "LanceDB BigInt numeric handling issue", "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations." + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" +} +\`\`\` + +## constructed register (game / roleplay / hypothetical) +\`\`\`json +{ + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "constructed" } \`\`\` +Note: the invented house rule and the bet itself are NOT extracted as separate preferences/cases — only this one episodic note survives. # Output Format @@ -119,7 +148,8 @@ Return JSON: "category": "profile|preferences|entities|events|cases|patterns", "abstract": "One-line index", "overview": "Structured Markdown summary", - "content": "Full narrative" + "content": "Full narrative", + "grounding": "real|constructed" } ] } @@ -129,7 +159,8 @@ Notes: - Only extract truly valuable personalized information - If nothing worth recording, return {"memories": []} - Maximum 5 memories per extraction -- Preferences should be aggregated by topic`; +- Preferences should be aggregated by topic +- Tag every memory's "grounding" field per the Conversational Grounding rules above`; } export function buildDedupPrompt(candidateAbstract, candidateOverview, candidateContent, existingMemories) { return `Determine how to handle this candidate memory. diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 1bd45d99f..068663b77 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -154,6 +154,28 @@ export function stripEnvelopeMetadata(text) { cleaned = cleaned.replace(/\n{3,}/g, "\n\n"); return cleaned.trim(); } +function globToRegExp(glob) { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + return new RegExp(`^${escaped}$`); +} +/** + * Resolve the extraction policy for a scope against a scope-glob -> mode map. + * Exact-string entries take priority over glob entries; an unmatched scope + * (or an absent policy map) defaults to "full". + */ +export function resolveExtractionPolicy(scope, policy) { + if (!policy) + return "full"; + if (Object.prototype.hasOwnProperty.call(policy, scope)) { + return policy[scope]; + } + for (const [glob, mode] of Object.entries(policy)) { + if (glob.includes("*") && globToRegExp(glob).test(scope)) { + return mode; + } + } + return "full"; +} // ============================================================================ // Constants // ============================================================================ @@ -244,8 +266,15 @@ export class SmartExtractor { ? options.scopeFilter : [targetScope]; const agentId = options.agentId; + // Option C: scope-glob extraction policy — "none" skips extraction + // entirely, with zero LLM calls, before grounding is ever considered. + const policyMode = resolveExtractionPolicy(targetScope, this.config.extractionPolicy); + if (policyMode === "none") { + this.log(`memory-pro: smart-extractor: extraction policy "none" for scope ${targetScope}, skipping extraction`); + return stats; + } // Step 1: LLM extraction - const extraction = await this.extractCandidates(conversationText); + const extraction = await this.extractCandidates(conversationText, policyMode); const candidates = extraction.candidates; if (candidates.length === 0) { this.log("memory-pro: smart-extractor: no memories extracted"); @@ -515,7 +544,7 @@ export class SmartExtractor { /** * Call LLM to extract candidate memories from conversation text. */ - async extractCandidates(conversationText) { + async extractCandidates(conversationText, policyMode = "full") { const maxChars = this.config.extractMaxChars ?? 8000; const truncated = conversationText.length > maxChars ? conversationText.slice(-maxChars) @@ -541,6 +570,9 @@ export class SmartExtractor { let invalidCategoryCount = 0; let shortAbstractCount = 0; let noiseAbstractCount = 0; + let policyDroppedCount = 0; + let constructedDroppedCount = 0; + let constructedEpisodicUsed = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -567,9 +599,37 @@ export class SmartExtractor { this.debugLog(`memory-lancedb-pro: smart-extractor: dropping candidate due to noise abstract category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); continue; } + // Option C: scope policy restricts extraction to episodic-only, + // independent of grounding — checked before the grounding filter below. + if (policyMode === "episodic-only" && category !== "events") { + policyDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping candidate due to episodic-only extraction policy category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + // Option A: grounding-aware register filter. Missing/non-string/unrecognized + // values fail open to "real" so a model that ignores the field can't break + // extraction. Constructed content (games, roleplay, fiction, hypotheticals, + // sample data) never becomes a durable profile/preferences/entities/cases/ + // patterns memory; at most one constructed "events" note per extraction + // records that the real participants did the activity. + const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; + const grounding = rawGrounding === "constructed" ? "constructed" : "real"; + if (grounding === "constructed") { + if (category !== "events") { + constructedDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate outside events category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + if (constructedEpisodicUsed) { + constructedDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping extra constructed-grounding events candidate (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + constructedEpisodicUsed = true; + } candidates.push({ category, abstract, overview, content }); } - this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}`); + this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}`); return { status: "ok", candidates }; } // -------------------------------------------------------------------------- From 696cf6a3451a9e68234c6c3365513350a7df82a9 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 12 Jul 2026 19:11:29 +0300 Subject: [PATCH 06/11] fix(extraction): add batch register signal and grounding-aware admission The per-item grounding self-tags wobble: the same clean fixture produced all-real tags on one run and correct constructed tags on two others, so a filter that trusts per-item tags alone (failing open to real) lets mislabeled fiction land in durable registers. Counter this at three deterministic layers plus the admission judge: - Extraction prompt v2: the model judges a batch-level conversation_register (real, mixed, fiction) once per extraction, and the grounding rules now state that grounding is judged per item with no expected count per batch, that the one-constructed-events cap is a storage rule rather than a tagging quota, and that items sharing a fictional frame must share a grounding tag. Few-shot examples now show full batches in varied shapes, including an all-constructed mid-game batch and a mixed batch with a real out-of-character aside. - Filter enforcement: register fiction drops all durable candidates regardless of per-item tags and keeps at most one events note; a mixed or missing register runs a batch contradiction check that demotes real-tagged durables whenever a sibling candidate carries an explicit constructed tag; register real keeps current behavior. Missing registers fail toward scrutiny (mixed), never toward open. - Grounding propagation: candidates keep their grounding and register past the filter, into stored metadata (grounding, conversation_register) and into the admission audit record, creating a persisted trail of which stored rows used the constructed allowance. - Admission: constructed candidates targeting durable registers are rejected deterministically before any LLM call; the utility prompt now interpolates grounding and register, names all six memory registers correctly, and instructs near-zero durable scores for fiction-framed content; the type prior for durable categories is capped at the events prior when the batch register is fiction, so the 0.6-weight prior cannot launder fiction into profile or preferences. Legacy payloads without grounding metadata keep their existing behavior throughout. Red-proved: 9 of the new regression tests fail against the previous implementation and pass after it. --- src/admission-control.ts | 90 ++++++- src/extraction-prompts.ts | 119 ++++++--- src/memory-categories.ts | 24 ++ src/smart-extractor.ts | 85 +++++- test/extraction-grounding-register.test.mjs | 276 +++++++++++++++++++- 5 files changed, 547 insertions(+), 47 deletions(-) diff --git a/src/admission-control.ts b/src/admission-control.ts index eee44d35e..8cec91cde 100644 --- a/src/admission-control.ts +++ b/src/admission-control.ts @@ -1,6 +1,6 @@ import { join } from "node:path"; import type { LlmClient } from "./llm-client.js"; -import type { CandidateMemory, MemoryCategory } from "./memory-categories.js"; +import { DURABLE_CATEGORIES, type CandidateMemory, type MemoryCategory } from "./memory-categories.js"; import type { MemorySearchResult, MemoryStore } from "./store.js"; import { parseSmartMetadata } from "./smart-metadata.js"; @@ -70,6 +70,10 @@ export interface AdmissionAuditRecord { compared_existing_memory_ids: string[]; max_similarity: number; evaluated_at: number; + /** Candidate's extraction-time grounding self-tag; absent on legacy payloads. */ + grounding?: "real" | "constructed"; + /** Batch register the candidate was extracted under; absent on legacy payloads. */ + conversation_register?: "real" | "mixed" | "fiction"; } export interface AdmissionEvaluation { @@ -474,12 +478,19 @@ Candidate memory: - Abstract: ${candidate.abstract} - Overview: ${candidate.overview} - Content: ${candidate.content} +- Grounding: ${candidate.grounding ?? "unknown (legacy payload, treat as real)"} +- Conversation register: ${candidate.conversationRegister ?? "unknown (legacy payload)"} Score future usefulness on a 0.0-1.0 scale. -Use higher scores for durable preferences, profile facts, reusable procedures, and long-lived project/entity state. +The memory system stores six categories: profile (user identity), preferences (user tendencies), entities (long-lived project/entity state), events (things that happened), cases (problem + solution pairs), and patterns (reusable procedures). + +Use higher scores for durable profile facts, preferences, entity state, patterns, and genuinely reusable cases. +Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. +Grounding rule: content asserted only inside roleplay, a game, fiction, a hypothetical, or a simulation is not a fact about the real user. If the excerpt shows a constructed frame (game rules, personas, "let's play", "suppose", canon of an invented world) and the candidate's claim lives inside that frame, score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity may still score moderately. + Return JSON only: { "utility": 0.0, @@ -512,6 +523,28 @@ export function scoreTypePrior( return clamp01(typePriors[category], DEFAULT_TYPE_PRIORS[category]); } +/** + * Grounding-aware type prior. The raw prior gives durable registers a large + * head start (default weights put 0.6 on this single feature); when the batch + * register says the conversation was fiction, that head start would launder a + * mislabeled in-fiction claim into profile/preferences. Cap the prior for + * durable categories at the events prior in that case; every other input + * keeps the raw prior untouched. + */ +export function scoreGroundedTypePrior( + candidate: CandidateMemory, + typePriors: AdmissionTypePriors, +): number { + const raw = scoreTypePrior(candidate.category, typePriors); + if ( + candidate.conversationRegister === "fiction" && + DURABLE_CATEGORIES.has(candidate.category) + ) { + return Math.min(raw, clamp01(typePriors.events, DEFAULT_TYPE_PRIORS.events)); + } + return raw; +} + export function scoreConfidenceSupport( candidate: CandidateMemory, conversationText: string, @@ -635,6 +668,42 @@ export class AdmissionController { private readonly debugLog: (msg: string) => void = () => {}, ) {} + private rejectConstructedDurable( + candidate: CandidateMemory, + now: number, + ): AdmissionEvaluation { + const featureScores: AdmissionFeatureScores = { + utility: 0, + confidence: 0, + novelty: 0, + recency: 0, + typePrior: 0, + }; + const reason = `Admission rejected (constructed-grounding candidate targeting durable category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const audit: AdmissionAuditRecord = { + version: "amac-v1", + decision: "reject", + score: 0, + reason, + thresholds: { + reject: this.config.rejectThreshold, + admit: this.config.admitThreshold, + }, + weights: this.config.weights, + feature_scores: featureScores, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0, + evaluated_at: now, + grounding: candidate.grounding, + conversation_register: candidate.conversationRegister, + }; + this.debugLog( + `memory-lancedb-pro: admission-control: decision=reject (constructed durable short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`, + ); + return { decision: "reject", audit }; + } + private async loadRelevantMatches( candidate: CandidateMemory, candidateVector: number[], @@ -671,6 +740,19 @@ export class AdmissionController { now?: number; }): Promise { const now = params.now ?? Date.now(); + + // Deterministic pre-admission short-circuit: a candidate tagged + // "constructed" must never occupy a durable register, no matter how any + // downstream score would blend. Reject before any LLM call. Candidates + // without grounding metadata (legacy payloads) fall through to normal + // scoring. + if ( + params.candidate.grounding === "constructed" && + DURABLE_CATEGORIES.has(params.candidate.category) + ) { + return this.rejectConstructedDurable(params.candidate, now); + } + const relevantMatches = await this.loadRelevantMatches( params.candidate, params.candidateVector, @@ -686,7 +768,7 @@ export class AdmissionController { const confidence = scoreConfidenceSupport(params.candidate, params.conversationText); const novelty = scoreNoveltyFromMatches(params.candidateVector, relevantMatches); const recency = scoreRecencyGap(now, relevantMatches, this.config.recency.halfLifeDays); - const typePrior = scoreTypePrior(params.candidate.category, this.config.typePriors); + const typePrior = scoreGroundedTypePrior(params.candidate, this.config.typePriors); const featureScores: AdmissionFeatureScores = { utility: utility.score, @@ -737,6 +819,8 @@ export class AdmissionController { compared_existing_memory_ids: novelty.comparedIds, max_similarity: novelty.maxSimilarity, evaluated_at: now, + grounding: params.candidate.grounding, + conversation_register: params.candidate.conversationRegister, }; this.debugLog( diff --git a/src/extraction-prompts.ts b/src/extraction-prompts.ts index eeebbd23c..c18013839 100644 --- a/src/extraction-prompts.ts +++ b/src/extraction-prompts.ts @@ -35,10 +35,10 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete fact/preference/decision/entity from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile fact, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. -- Atomic memory shape: each stored memory must read like one durable fact, preference, decision, entity state, event, case, or reusable pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. +- Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. - Length/distillation gate: if a candidate is longer than about 200 characters and reads like raw conversation instead of a distilled insight, rewrite it as a single factual statement before storing; if that is not possible, skip it. # Memory Classification @@ -72,7 +72,21 @@ ${conversationText} # Conversational Grounding -Every memory must also be tagged with how grounded it is in reality: +First judge the register of the WHOLE conversation once, then tag each memory item independently. + +## Batch register (one judgment per extraction) + +Set the top-level "conversation_register" field: + +| conversation_register | Meaning | +|-----------------------|---------| +| "real" | An ordinary working/personal conversation about the actual user and the real world | +| "fiction" | The conversation is inside a constructed frame: a game in progress, roleplay, in-character dialogue, drafted fiction, a hypothetical scenario, or a sample-data exercise | +| "mixed" | A constructed frame AND genuine out-of-character content are interleaved | + +## Per-item grounding + +Tag every memory's "grounding" field: | grounding | Meaning | |-----------|---------| @@ -80,9 +94,12 @@ Every memory must also be tagged with how grounded it is in reality: | "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | Rules: -- When the surrounding register is a game, roleplay, fiction, hypothetical, or sample-data exercise, you may record AT MOST ONE session-scoped "events" memory noting that the real participants did the activity (e.g. "agent-one and agent-two ran a puzzle exercise"). Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. +- Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. +- If the conversation is in-character or mid-fiction, EVERY item derived from the narrative is "constructed" — an all-constructed batch is the normal result for such input. +- Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. From a constructed frame, the only extractable memory is a session-scoped "events" note that the real participants did the activity; the storage layer keeps at most one such note per extraction. That cap is a storage rule applied after tagging — it is NOT a tagging quota, so never re-tag extra items "real" to fit under it. - A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. -- If you are unsure, default to "real" — under-tagging as constructed risks losing a genuine fact. +- Self-consistency check before answering: items derived from the same fictional frame must share the same grounding tag. If your draft tags one game-derived item "constructed" and another game-derived item "real", re-check the batch — either the item is a genuine out-of-character aside, or its tag is wrong. +- If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. # Three-Level Structure @@ -98,55 +115,93 @@ Each memory contains three levels: # Few-shot Examples -## profile +Each example is a full output batch, because register and grounding are judged together. + +## Ordinary working conversation (register "real", single memory) \`\`\`json { - "category": "profile", - "abstract": "User basic info: AI development engineer, 3 years LLM experience", - "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience.", - "grounding": "real" + "conversation_register": "real", + "memories": [ + { + "category": "cases", + "abstract": "LanceDB BigInt numeric handling issue", + "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" + } + ] } \`\`\` -## preferences +## Ordinary personal conversation (register "real", two memories) \`\`\`json { - "category": "preferences", - "abstract": "Python code style: No type hints, concise and direct", - "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments.", - "grounding": "real" + "conversation_register": "real", + "memories": [ + { + "category": "profile", + "abstract": "User basic info: AI development engineer, 3 years LLM experience", + "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" + }, + { + "category": "preferences", + "abstract": "Python code style: No type hints, concise and direct", + "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" + } + ] } \`\`\` -## cases +## Mid-game conversation (register "fiction", ALL items constructed) +Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { - "category": "cases", - "abstract": "LanceDB BigInt numeric handling issue", - "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", - "grounding": "real" + "conversation_register": "fiction", + "memories": [ + { + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "constructed" + } + ] } \`\`\` +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. The whole batch carries "constructed" tags; that is the expected shape for mid-fiction input. -## constructed register (game / roleplay / hypothetical) +## Game with a genuine out-of-character aside (register "mixed") \`\`\`json { - "category": "events", - "abstract": "agent-one and agent-two ran a two-round puzzle exercise", - "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", - "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", - "grounding": "constructed" + "conversation_register": "mixed", + "memories": [ + { + "category": "events", + "abstract": "User mentioned their new laptop arrives Thursday", + "overview": "## Real-world aside\\n- Stated in passing during a game", + "content": "In the middle of the game the user mentioned, out of character, that their new laptop arrives Thursday.", + "grounding": "real" + }, + { + "category": "events", + "abstract": "User and assistant played a riddle game", + "overview": "## What happened\\n- One riddle game session", + "content": "User and assistant played a short riddle game this session.", + "grounding": "constructed" + } + ] } \`\`\` -Note: the invented house rule and the bet itself are NOT extracted as separate preferences/cases — only this one episodic note survives. # Output Format Return JSON: { + "conversation_register": "real|mixed|fiction", "memories": [ { "category": "profile|preferences|entities|events|cases|patterns", @@ -161,10 +216,10 @@ Return JSON: Notes: - Output language should match the dominant language in the conversation - Only extract truly valuable personalized information -- If nothing worth recording, return {"memories": []} +- If nothing worth recording, return {"conversation_register": "real|mixed|fiction", "memories": []} - Maximum 5 memories per extraction - Preferences should be aggregated by topic -- Tag every memory's "grounding" field per the Conversational Grounding rules above`; +- Always set the top-level "conversation_register" field, and tag every memory's "grounding" field, per the Conversational Grounding rules above`; } export function buildDedupPrompt( diff --git a/src/memory-categories.ts b/src/memory-categories.ts index 809c0327f..ca5fdf243 100644 --- a/src/memory-categories.ts +++ b/src/memory-categories.ts @@ -80,12 +80,36 @@ export const APPEND_ONLY_CATEGORIES = new Set([ /** Memory tier levels for lifecycle management. */ export type MemoryTier = "core" | "working" | "peripheral"; +/** Per-candidate conversational grounding self-tag from extraction. */ +export type CandidateGrounding = "real" | "constructed"; + +/** + * Batch-level register judgment for the whole extraction input: + * "real" ordinary conversation, "fiction" an in-character/game/roleplay + * frame, "mixed" both interleaved. Judged once per extraction batch — + * more stable than the per-item grounding tags. + */ +export type ConversationRegister = "real" | "mixed" | "fiction"; + +/** Durable registers: constructed content must never land in these. */ +export const DURABLE_CATEGORIES = new Set([ + "profile", + "preferences", + "entities", + "cases", + "patterns", +]); + /** A candidate memory extracted from conversation by LLM. */ export type CandidateMemory = { category: MemoryCategory; abstract: string; // L0: one-sentence index overview: string; // L1: structured markdown summary content: string; // L2: full narrative + /** Absent on legacy payloads: treat as "real" (fail open per item). */ + grounding?: CandidateGrounding; + /** Batch register the candidate was extracted under; absent on legacy payloads. */ + conversationRegister?: ConversationRegister; }; /** Dedup decision from LLM. */ diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index ced1975fb..d86c4313b 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -21,12 +21,15 @@ import { type AdmissionRejectionAuditEntry, } from "./admission-control.js"; import { + type CandidateGrounding, type CandidateMemory, + type ConversationRegister, type DedupDecision, type DedupResult, type ExtractionStats, type MemoryCategory, ALWAYS_MERGE_CATEGORIES, + DURABLE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, MEMORY_CATEGORIES, @@ -823,6 +826,7 @@ export class SmartExtractor { const prompt = buildExtractionPrompt(cleaned, user); const result = await this.llm.completeJson<{ + conversation_register?: string; memories: Array<{ category: string; abstract: string; @@ -849,6 +853,18 @@ export class SmartExtractor { `memory-lancedb-pro: smart-extractor: extract-candidates raw memories=${result.memories.length}`, ); + // Batch-level register signal, judged once per extraction. The model + // classifies whole sessions far more reliably than it self-tags single + // items, so the register deterministically overrides per-item grounding + // wobble below. Missing/unrecognized values fail toward scrutiny + // ("mixed"), never toward open. + const rawRegister = + typeof result.conversation_register === "string" + ? result.conversation_register.toLowerCase().trim() + : ""; + const conversationRegister: ConversationRegister = + rawRegister === "real" || rawRegister === "fiction" ? rawRegister : "mixed"; + // Validate and normalize candidates const candidates: CandidateMemory[] = []; let invalidCategoryCount = 0; @@ -856,7 +872,9 @@ export class SmartExtractor { let noiseAbstractCount = 0; let policyDroppedCount = 0; let constructedDroppedCount = 0; + let fictionRegisterDroppedCount = 0; let constructedEpisodicUsed = false; + let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -905,16 +923,40 @@ export class SmartExtractor { } // Option A: grounding-aware register filter. Missing/non-string/unrecognized - // values fail open to "real" so a model that ignores the field can't break - // extraction. Constructed content (games, roleplay, fiction, hypotheticals, - // sample data) never becomes a durable profile/preferences/entities/cases/ - // patterns memory; at most one constructed "events" note per extraction - // records that the real participants did the activity. + // per-item values fail open to "real" so a model that ignores the field + // can't break extraction. Constructed content (games, roleplay, fiction, + // hypotheticals, sample data) never becomes a durable profile/preferences/ + // entities/cases/patterns memory; at most one constructed "events" note per + // extraction records that the real participants did the activity. const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; - const grounding: "real" | "constructed" = + const grounding: CandidateGrounding = rawGrounding === "constructed" ? "constructed" : "real"; - if (grounding === "constructed") { + if (rawGrounding === "constructed") { + batchHasConstructedTag = true; + } + + if (conversationRegister === "fiction") { + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override), and + // keeps at most one session-scoped events note. + if (DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + if (constructedEpisodicUsed) { + fictionRegisterDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping extra events candidate from fiction-register batch (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + constructedEpisodicUsed = true; + } else if (grounding === "constructed") { if (category !== "events") { constructedDroppedCount++; this.debugLog( @@ -932,11 +974,30 @@ export class SmartExtractor { constructedEpisodicUsed = true; } - candidates.push({ category, abstract, overview, content }); + candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); + } + + // Batch contradiction check: when the batch shows evidence of a + // constructed frame (register not "real" AND at least one item the model + // itself tagged constructed), surviving real-tagged durables in the same + // batch are the classic mislabel shape — fiction items where only the + // grounding tag wobbled. Demote them deterministically, no extra LLM call. + let contradictionDemotedCount = 0; + if (conversationRegister !== "real" && batchHasConstructedTag) { + for (let i = candidates.length - 1; i >= 0; i--) { + const candidate = candidates[i]; + if (DURABLE_CATEGORIES.has(candidate.category)) { + contradictionDemotedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: demoting real-tagged durable from ${conversationRegister}-register batch with constructed siblings (batch contradiction) category=${candidate.category} abstract=${JSON.stringify(candidate.abstract.slice(0, 120))}`, + ); + candidates.splice(i, 1); + } + } } this.debugLog( - `memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}`, + `memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`, ); return { status: "ok", candidates }; @@ -1832,6 +1893,12 @@ export class SmartExtractor { suppressed_until_turn: 0, memory_temporal_type: classifyTemporal(classifyText), valid_until: inferExpiry(classifyText), + // Grounding audit trail: which self-tag and batch register this + // memory was admitted under. Absent on legacy payloads. + ...(candidate.grounding ? { grounding: candidate.grounding } : {}), + ...(candidate.conversationRegister + ? { conversation_register: candidate.conversationRegister } + : {}), ...(admissionAudit ? { admission_audit: JSON.stringify(admissionAudit) } : {}), }, ), diff --git a/test/extraction-grounding-register.test.mjs b/test/extraction-grounding-register.test.mjs index 13e0a47e2..88e025309 100644 --- a/test/extraction-grounding-register.test.mjs +++ b/test/extraction-grounding-register.test.mjs @@ -55,14 +55,20 @@ function makeEmbedder(dims = 97) { }; } -/** `memories` is the raw memories array the mocked "extract-candidates" LLM call returns. */ -function makeLlm(memories) { +/** + * `memories` is the raw memories array the mocked "extract-candidates" LLM + * call returns; `conversationRegister` (optional) is the batch-level + * conversation_register field — omitted to simulate legacy payloads. + */ +function makeLlm(memories, conversationRegister) { let extractCandidatesCalls = 0; return { async completeJson(_prompt, mode) { if (mode !== "extract-candidates") return null; extractCandidatesCalls++; - return { memories }; + return conversationRegister + ? { conversation_register: conversationRegister, memories } + : { memories }; }, get extractCandidatesCalls() { return extractCandidatesCalls; @@ -312,3 +318,267 @@ describe("SmartExtractor scope-glob extraction policy (Option C)", () => { assert.equal(stats.created, 2); }); }); + +// ============================================================================ +// Grounding v2 — batch register signal, contradiction check, propagation, +// grounding-aware admission +// ============================================================================ + +/** Fiction-frame durables the extractor mislabeled as "real" (Mode A shape). */ +const MISLABELED_FICTION_CANDIDATES = [ + { + category: "profile", + abstract: "User lives in Moon Base 9", + overview: "## Background\n- Residence: Moon Base 9", + content: "User lives in Moon Base 9.", + grounding: "real", // wrong: asserted only inside the game canon + }, + { + category: "preferences", + abstract: "Favorite drink is nebula tea", + overview: "## Preference\n- Drink: nebula tea", + content: "User's favorite drink is nebula tea.", + grounding: "real", // wrong: asserted only inside the game canon + }, + { + category: "events", + abstract: "User and assistant played one round of a space roleplay game", + overview: "## What happened\n- One roleplay round", + content: "User and assistant played one round of a space roleplay game this session.", + grounding: "constructed", + }, +]; + +describe("SmartExtractor batch register signal (grounding v2)", () => { + it("register 'fiction' drops ALL durable candidates even when their per-item tags say 'real'", async () => { + const store = makeStore(); + const llm = makeLlm(MISLABELED_FICTION_CANDIDATES, "fiction"); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual( + categories, + ["events"], + "mislabeled real-tagged durables must not survive a fiction-register batch", + ); + assert.equal(stats.created, 1); + }); + + it("register 'fiction' caps events notes at one, regardless of per-item tags", async () => { + const store = makeStore(); + const llm = makeLlm( + [ + MISLABELED_FICTION_CANDIDATES[2], + { + ...MISLABELED_FICTION_CANDIDATES[2], + abstract: "User and assistant also played a bonus round", + grounding: "real", // tag wobble: still capped by the register rule + }, + ], + "fiction", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + assert.equal(persistedCategories(store).length, 1); + }); + + it("register 'mixed' with a constructed sibling demotes real-tagged durables (batch contradiction check)", async () => { + const store = makeStore(); + const llm = makeLlm( + [ + MISLABELED_FICTION_CANDIDATES[0], // profile tagged real + MISLABELED_FICTION_CANDIDATES[2], // events tagged constructed — the contradiction evidence + REAL_ASIDE_CANDIDATE, // real events aside must still survive + ], + "mixed", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.ok(!categories.includes("profile"), "real-tagged durable must be demoted in a mixed-register batch with constructed siblings"); + assert.equal(categories.filter((c) => c === "events").length, 2, "both events notes (constructed episodic + real aside) survive"); + }); + + it("register 'real' fully trusts per-item tags (durables kept alongside a constructed events note)", async () => { + const store = makeStore(); + const llm = makeLlm( + [FACTUAL_CANDIDATES[0], MISLABELED_FICTION_CANDIDATES[2]], + "real", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist("mostly factual conversation with a brief game", "s1"); + + const categories = persistedCategories(store).sort(); + assert.deepEqual(categories, ["events", "preferences"], "register 'real' must not demote per-item-real durables"); + }); + + it("missing register (legacy payload) with no constructed tags behaves exactly as before", async () => { + const store = makeStore(); + const llm = makeLlm(FACTUAL_CANDIDATES); // no conversation_register field + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist("some factual conversation", "s1"); + + assert.equal(stats.created, 2, "legacy payloads (no register, no constructed tags) must be unaffected"); + }); + + it("propagates grounding and conversation_register into stored metadata", async () => { + const store = makeStore(); + const llm = makeLlm( + [FACTUAL_CANDIDATES[0], MISLABELED_FICTION_CANDIDATES[2]], + "real", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + await extractor.extractAndPersist("mostly factual conversation with a brief game", "s1"); + + const metas = store.bulkStoreCalls.flat().map((e) => JSON.parse(e.metadata || "{}")); + const prefMeta = metas.find((m) => m.memory_category === "preferences"); + const eventsMeta = metas.find((m) => m.memory_category === "events"); + assert.equal(prefMeta.grounding, "real"); + assert.equal(prefMeta.conversation_register, "real"); + assert.equal(eventsMeta.grounding, "constructed"); + assert.equal(eventsMeta.conversation_register, "real"); + }); + + it("buildExtractionPrompt documents the batch register contract (structural check)", () => { + const { buildExtractionPrompt } = jiti("../src/extraction-prompts.ts"); + const prompt = buildExtractionPrompt("some conversation", "test-user"); + + assert.match(prompt, /conversation_register/); + assert.match(prompt, /"real\|mixed\|fiction"/); + assert.match(prompt, /storage rule applied after tagging/i, "the cap must be framed as a storage rule, not a tagging quota"); + assert.match(prompt, /all-constructed batch/i, "all-constructed batches must be documented as normal"); + assert.match(prompt, /self-consistency/i, "the batch self-consistency instruction must be present"); + }); +}); + +describe("AdmissionController grounding awareness (grounding v2)", () => { + const { AdmissionController, ADMISSION_CONTROL_PRESETS, scoreGroundedTypePrior } = jiti("../src/admission-control.ts"); + const balanced = ADMISSION_CONTROL_PRESETS.balanced; + + function makeAdmissionLlm() { + let utilityCalls = 0; + return { + async completeJson(_prompt, mode) { + if (mode === "admission-utility") { + utilityCalls++; + return { utility: 0.9, reason: "mock" }; + } + return null; + }, + get utilityCalls() { + return utilityCalls; + }, + }; + } + + const admissionStore = { async vectorSearch() { return []; } }; + + it("short-circuits constructed durable candidates to reject with zero LLM calls", async () => { + const llm = makeAdmissionLlm(); + const controller = new AdmissionController(admissionStore, llm, balanced); + + const evaluation = await controller.evaluate({ + candidate: { + category: "preferences", + abstract: "House rule: puzzle answers must be even numbers", + overview: "## Rule\n- Even numbers only", + content: "The puzzle house rule is that answers must always be even numbers.", + grounding: "constructed", + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(evaluation.decision, "reject"); + assert.equal(llm.utilityCalls, 0, "the short-circuit must not spend an LLM call"); + assert.match(evaluation.audit.reason, /constructed-grounding/); + assert.equal(evaluation.audit.grounding, "constructed"); + assert.equal(evaluation.audit.conversation_register, "fiction"); + }); + + it("constructed events candidates still go through normal scoring (only durables short-circuit)", async () => { + const llm = makeAdmissionLlm(); + const controller = new AdmissionController(admissionStore, llm, balanced); + + const evaluation = await controller.evaluate({ + candidate: { + category: "events", + abstract: "User and assistant played one round of a space roleplay game", + overview: "## What happened\n- One roleplay round", + content: "User and assistant played one round of a space roleplay game this session.", + grounding: "constructed", + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(llm.utilityCalls, 1, "events candidates use the normal scoring path"); + assert.equal(evaluation.audit.grounding, "constructed"); + }); + + it("scoreGroundedTypePrior caps durable priors at the events prior for fiction-register candidates", () => { + const priors = balanced.typePriors; + const fictionProfile = { + category: "profile", + abstract: "x", + overview: "", + content: "", + grounding: "real", + conversationRegister: "fiction", + }; + const realProfile = { ...fictionProfile, conversationRegister: "real" }; + + assert.equal(scoreGroundedTypePrior(fictionProfile, priors), priors.events, "fiction register must cap the profile prior at the events prior"); + assert.equal(scoreGroundedTypePrior(realProfile, priors), priors.profile, "real register keeps the raw prior"); + }); + + it("buildUtilityPrompt interpolates grounding and names all six registers (structural check)", async () => { + const llm = { + prompts: [], + async completeJson(prompt, mode) { + if (mode === "admission-utility") { + this.prompts.push(prompt); + return { utility: 0.5, reason: "mock" }; + } + return null; + }, + }; + const controller = new AdmissionController(admissionStore, llm, balanced); + + await controller.evaluate({ + candidate: { + category: "events", + abstract: "User and assistant played one round of a space roleplay game", + overview: "", + content: "One roleplay round happened.", + grounding: "constructed", + conversationRegister: "fiction", + }, + candidateVector: [1, 0, 0], + conversationText: GAME_TRANSCRIPT, + scopeFilter: ["global"], + }); + + assert.equal(llm.prompts.length, 1); + const prompt = llm.prompts[0]; + assert.match(prompt, /Grounding: constructed/); + assert.match(prompt, /Conversation register: fiction/); + for (const register of ["profile", "preferences", "entities", "events", "cases", "patterns"]) { + assert.match(prompt, new RegExp(register), `utility prompt must name the ${register} register`); + } + assert.match(prompt, /roleplay/i, "utility prompt must carry the fiction guidance"); + }); +}); From 52f821179e74e60ab36892f9a9c6556e70d9e000 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 12 Jul 2026 19:11:34 +0300 Subject: [PATCH 07/11] chore: rebuild dist for batch register signal and grounding-aware admission --- dist/src/admission-control.js | 69 ++++++++++++++++++- dist/src/extraction-prompts.js | 119 ++++++++++++++++++++++++--------- dist/src/memory-categories.js | 8 +++ dist/src/smart-extractor.js | 71 +++++++++++++++++--- 4 files changed, 224 insertions(+), 43 deletions(-) diff --git a/dist/src/admission-control.js b/dist/src/admission-control.js index 44733bb58..6371a780a 100644 --- a/dist/src/admission-control.js +++ b/dist/src/admission-control.js @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { DURABLE_CATEGORIES } from "./memory-categories.js"; import { parseSmartMetadata } from "./smart-metadata.js"; const DEFAULT_WEIGHTS = { utility: 0.1, @@ -322,12 +323,19 @@ Candidate memory: - Abstract: ${candidate.abstract} - Overview: ${candidate.overview} - Content: ${candidate.content} +- Grounding: ${candidate.grounding ?? "unknown (legacy payload, treat as real)"} +- Conversation register: ${candidate.conversationRegister ?? "unknown (legacy payload)"} Score future usefulness on a 0.0-1.0 scale. -Use higher scores for durable preferences, profile facts, reusable procedures, and long-lived project/entity state. +The memory system stores six categories: profile (user identity), preferences (user tendencies), entities (long-lived project/entity state), events (things that happened), cases (problem + solution pairs), and patterns (reusable procedures). + +Use higher scores for durable profile facts, preferences, entity state, patterns, and genuinely reusable cases. +Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. +Grounding rule: content asserted only inside roleplay, a game, fiction, a hypothetical, or a simulation is not a fact about the real user. If the excerpt shows a constructed frame (game rules, personas, "let's play", "suppose", canon of an invented world) and the candidate's claim lives inside that frame, score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity may still score moderately. + Return JSON only: { "utility": 0.0, @@ -347,6 +355,22 @@ function buildReason(details) { export function scoreTypePrior(category, typePriors) { return clamp01(typePriors[category], DEFAULT_TYPE_PRIORS[category]); } +/** + * Grounding-aware type prior. The raw prior gives durable registers a large + * head start (default weights put 0.6 on this single feature); when the batch + * register says the conversation was fiction, that head start would launder a + * mislabeled in-fiction claim into profile/preferences. Cap the prior for + * durable categories at the events prior in that case; every other input + * keeps the raw prior untouched. + */ +export function scoreGroundedTypePrior(candidate, typePriors) { + const raw = scoreTypePrior(candidate.category, typePriors); + if (candidate.conversationRegister === "fiction" && + DURABLE_CATEGORIES.has(candidate.category)) { + return Math.min(raw, clamp01(typePriors.events, DEFAULT_TYPE_PRIORS.events)); + } + return raw; +} export function scoreConfidenceSupport(candidate, conversationText) { const candidateText = `${candidate.abstract}\n${candidate.content}`.trim(); const candidateTokens = tokenizeText(candidateText); @@ -437,6 +461,36 @@ export class AdmissionController { this.config = config; this.debugLog = debugLog; } + rejectConstructedDurable(candidate, now) { + const featureScores = { + utility: 0, + confidence: 0, + novelty: 0, + recency: 0, + typePrior: 0, + }; + const reason = `Admission rejected (constructed-grounding candidate targeting durable category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const audit = { + version: "amac-v1", + decision: "reject", + score: 0, + reason, + thresholds: { + reject: this.config.rejectThreshold, + admit: this.config.admitThreshold, + }, + weights: this.config.weights, + feature_scores: featureScores, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0, + evaluated_at: now, + grounding: candidate.grounding, + conversation_register: candidate.conversationRegister, + }; + this.debugLog(`memory-lancedb-pro: admission-control: decision=reject (constructed durable short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`); + return { decision: "reject", audit }; + } async loadRelevantMatches(candidate, candidateVector, scopeFilter) { if (!Array.isArray(candidateVector) || candidateVector.length === 0) { return []; @@ -453,12 +507,21 @@ export class AdmissionController { } async evaluate(params) { const now = params.now ?? Date.now(); + // Deterministic pre-admission short-circuit: a candidate tagged + // "constructed" must never occupy a durable register, no matter how any + // downstream score would blend. Reject before any LLM call. Candidates + // without grounding metadata (legacy payloads) fall through to normal + // scoring. + if (params.candidate.grounding === "constructed" && + DURABLE_CATEGORIES.has(params.candidate.category)) { + return this.rejectConstructedDurable(params.candidate, now); + } const relevantMatches = await this.loadRelevantMatches(params.candidate, params.candidateVector, params.scopeFilter); const utility = await scoreUtility(this.llm, this.config.utilityMode, params.candidate, params.conversationText); const confidence = scoreConfidenceSupport(params.candidate, params.conversationText); const novelty = scoreNoveltyFromMatches(params.candidateVector, relevantMatches); const recency = scoreRecencyGap(now, relevantMatches, this.config.recency.halfLifeDays); - const typePrior = scoreTypePrior(params.candidate.category, this.config.typePriors); + const typePrior = scoreGroundedTypePrior(params.candidate, this.config.typePriors); const featureScores = { utility: utility.score, confidence: confidence.score, @@ -502,6 +565,8 @@ export class AdmissionController { compared_existing_memory_ids: novelty.comparedIds, max_similarity: novelty.maxSimilarity, evaluated_at: now, + grounding: params.candidate.grounding, + conversation_register: params.candidate.conversationRegister, }; this.debugLog(`memory-lancedb-pro: admission-control: decision=${audit.decision} hint=${audit.hint ?? "n/a"} score=${audit.score.toFixed(3)} candidate=${JSON.stringify(params.candidate.abstract.slice(0, 80))}`); return { decision, hint, audit }; diff --git a/dist/src/extraction-prompts.js b/dist/src/extraction-prompts.js index 008f0a2da..8329f09ed 100644 --- a/dist/src/extraction-prompts.js +++ b/dist/src/extraction-prompts.js @@ -31,10 +31,10 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete fact/preference/decision/entity from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile fact, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. -- Atomic memory shape: each stored memory must read like one durable fact, preference, decision, entity state, event, case, or reusable pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. +- Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. - Length/distillation gate: if a candidate is longer than about 200 characters and reads like raw conversation instead of a distilled insight, rewrite it as a single factual statement before storing; if that is not possible, skip it. # Memory Classification @@ -68,7 +68,21 @@ ${conversationText} # Conversational Grounding -Every memory must also be tagged with how grounded it is in reality: +First judge the register of the WHOLE conversation once, then tag each memory item independently. + +## Batch register (one judgment per extraction) + +Set the top-level "conversation_register" field: + +| conversation_register | Meaning | +|-----------------------|---------| +| "real" | An ordinary working/personal conversation about the actual user and the real world | +| "fiction" | The conversation is inside a constructed frame: a game in progress, roleplay, in-character dialogue, drafted fiction, a hypothetical scenario, or a sample-data exercise | +| "mixed" | A constructed frame AND genuine out-of-character content are interleaved | + +## Per-item grounding + +Tag every memory's "grounding" field: | grounding | Meaning | |-----------|---------| @@ -76,9 +90,12 @@ Every memory must also be tagged with how grounded it is in reality: | "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | Rules: -- When the surrounding register is a game, roleplay, fiction, hypothetical, or sample-data exercise, you may record AT MOST ONE session-scoped "events" memory noting that the real participants did the activity (e.g. "agent-one and agent-two ran a puzzle exercise"). Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. +- Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. +- If the conversation is in-character or mid-fiction, EVERY item derived from the narrative is "constructed" — an all-constructed batch is the normal result for such input. +- Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. From a constructed frame, the only extractable memory is a session-scoped "events" note that the real participants did the activity; the storage layer keeps at most one such note per extraction. That cap is a storage rule applied after tagging — it is NOT a tagging quota, so never re-tag extra items "real" to fit under it. - A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. -- If you are unsure, default to "real" — under-tagging as constructed risks losing a genuine fact. +- Self-consistency check before answering: items derived from the same fictional frame must share the same grounding tag. If your draft tags one game-derived item "constructed" and another game-derived item "real", re-check the batch — either the item is a genuine out-of-character aside, or its tag is wrong. +- If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. # Three-Level Structure @@ -94,55 +111,93 @@ Each memory contains three levels: # Few-shot Examples -## profile +Each example is a full output batch, because register and grounding are judged together. + +## Ordinary working conversation (register "real", single memory) \`\`\`json { - "category": "profile", - "abstract": "User basic info: AI development engineer, 3 years LLM experience", - "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", - "content": "User is an AI development engineer with 3 years of LLM application development experience.", - "grounding": "real" + "conversation_register": "real", + "memories": [ + { + "category": "cases", + "abstract": "LanceDB BigInt numeric handling issue", + "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", + "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", + "grounding": "real" + } + ] } \`\`\` -## preferences +## Ordinary personal conversation (register "real", two memories) \`\`\`json { - "category": "preferences", - "abstract": "Python code style: No type hints, concise and direct", - "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", - "content": "User prefers Python code without type hints, with concise function comments.", - "grounding": "real" + "conversation_register": "real", + "memories": [ + { + "category": "profile", + "abstract": "User basic info: AI development engineer, 3 years LLM experience", + "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain", + "content": "User is an AI development engineer with 3 years of LLM application development experience.", + "grounding": "real" + }, + { + "category": "preferences", + "abstract": "Python code style: No type hints, concise and direct", + "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation", + "content": "User prefers Python code without type hints, with concise function comments.", + "grounding": "real" + } + ] } \`\`\` -## cases +## Mid-game conversation (register "fiction", ALL items constructed) +Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { - "category": "cases", - "abstract": "LanceDB BigInt numeric handling issue", - "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic", - "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations.", - "grounding": "real" + "conversation_register": "fiction", + "memories": [ + { + "category": "events", + "abstract": "agent-one and agent-two ran a two-round puzzle exercise", + "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", + "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", + "grounding": "constructed" + } + ] } \`\`\` +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. The whole batch carries "constructed" tags; that is the expected shape for mid-fiction input. -## constructed register (game / roleplay / hypothetical) +## Game with a genuine out-of-character aside (register "mixed") \`\`\`json { - "category": "events", - "abstract": "agent-one and agent-two ran a two-round puzzle exercise", - "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", - "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", - "grounding": "constructed" + "conversation_register": "mixed", + "memories": [ + { + "category": "events", + "abstract": "User mentioned their new laptop arrives Thursday", + "overview": "## Real-world aside\\n- Stated in passing during a game", + "content": "In the middle of the game the user mentioned, out of character, that their new laptop arrives Thursday.", + "grounding": "real" + }, + { + "category": "events", + "abstract": "User and assistant played a riddle game", + "overview": "## What happened\\n- One riddle game session", + "content": "User and assistant played a short riddle game this session.", + "grounding": "constructed" + } + ] } \`\`\` -Note: the invented house rule and the bet itself are NOT extracted as separate preferences/cases — only this one episodic note survives. # Output Format Return JSON: { + "conversation_register": "real|mixed|fiction", "memories": [ { "category": "profile|preferences|entities|events|cases|patterns", @@ -157,10 +212,10 @@ Return JSON: Notes: - Output language should match the dominant language in the conversation - Only extract truly valuable personalized information -- If nothing worth recording, return {"memories": []} +- If nothing worth recording, return {"conversation_register": "real|mixed|fiction", "memories": []} - Maximum 5 memories per extraction - Preferences should be aggregated by topic -- Tag every memory's "grounding" field per the Conversational Grounding rules above`; +- Always set the top-level "conversation_register" field, and tag every memory's "grounding" field, per the Conversational Grounding rules above`; } export function buildDedupPrompt(candidateAbstract, candidateOverview, candidateContent, existingMemories) { return `Determine how to handle this candidate memory. diff --git a/dist/src/memory-categories.js b/dist/src/memory-categories.js index abba922eb..edc745bef 100644 --- a/dist/src/memory-categories.js +++ b/dist/src/memory-categories.js @@ -58,6 +58,14 @@ export const APPEND_ONLY_CATEGORIES = new Set([ "events", "cases", ]); +/** Durable registers: constructed content must never land in these. */ +export const DURABLE_CATEGORIES = new Set([ + "profile", + "preferences", + "entities", + "cases", + "patterns", +]); /** Validate and normalize a category string. */ export function normalizeCategory(raw) { const lower = raw.toLowerCase().trim(); diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 068663b77..b96e624db 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -7,7 +7,7 @@ */ import { buildExtractionPrompt, buildDedupPrompt, buildMergePrompt, } from "./extraction-prompts.js"; import { AdmissionController, } from "./admission-control.js"; -import { ALWAYS_MERGE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; +import { ALWAYS_MERGE_CATEGORIES, DURABLE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; import { isMetaFrustrationNoise, isNoise } from "./noise-filter.js"; import { appendRelation, buildSmartMetadata, deriveFactKey, parseSmartMetadata, stringifySmartMetadata, parseSupportInfo, updateSupportStats, } from "./smart-metadata.js"; import { isUserMdExclusiveMemory, } from "./workspace-boundary.js"; @@ -565,6 +565,15 @@ export class SmartExtractor { return { status: "malformed", candidates: [] }; } this.debugLog(`memory-lancedb-pro: smart-extractor: extract-candidates raw memories=${result.memories.length}`); + // Batch-level register signal, judged once per extraction. The model + // classifies whole sessions far more reliably than it self-tags single + // items, so the register deterministically overrides per-item grounding + // wobble below. Missing/unrecognized values fail toward scrutiny + // ("mixed"), never toward open. + const rawRegister = typeof result.conversation_register === "string" + ? result.conversation_register.toLowerCase().trim() + : ""; + const conversationRegister = rawRegister === "real" || rawRegister === "fiction" ? rawRegister : "mixed"; // Validate and normalize candidates const candidates = []; let invalidCategoryCount = 0; @@ -572,7 +581,9 @@ export class SmartExtractor { let noiseAbstractCount = 0; let policyDroppedCount = 0; let constructedDroppedCount = 0; + let fictionRegisterDroppedCount = 0; let constructedEpisodicUsed = false; + let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { invalidCategoryCount++; @@ -607,14 +618,34 @@ export class SmartExtractor { continue; } // Option A: grounding-aware register filter. Missing/non-string/unrecognized - // values fail open to "real" so a model that ignores the field can't break - // extraction. Constructed content (games, roleplay, fiction, hypotheticals, - // sample data) never becomes a durable profile/preferences/entities/cases/ - // patterns memory; at most one constructed "events" note per extraction - // records that the real participants did the activity. + // per-item values fail open to "real" so a model that ignores the field + // can't break extraction. Constructed content (games, roleplay, fiction, + // hypotheticals, sample data) never becomes a durable profile/preferences/ + // entities/cases/patterns memory; at most one constructed "events" note per + // extraction records that the real participants did the activity. const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; const grounding = rawGrounding === "constructed" ? "constructed" : "real"; - if (grounding === "constructed") { + if (rawGrounding === "constructed") { + batchHasConstructedTag = true; + } + if (conversationRegister === "fiction") { + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override), and + // keeps at most one session-scoped events note. + if (DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + if (constructedEpisodicUsed) { + fictionRegisterDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping extra events candidate from fiction-register batch (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; + } + constructedEpisodicUsed = true; + } + else if (grounding === "constructed") { if (category !== "events") { constructedDroppedCount++; this.debugLog(`memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate outside events category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); @@ -627,9 +658,25 @@ export class SmartExtractor { } constructedEpisodicUsed = true; } - candidates.push({ category, abstract, overview, content }); + candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); + } + // Batch contradiction check: when the batch shows evidence of a + // constructed frame (register not "real" AND at least one item the model + // itself tagged constructed), surviving real-tagged durables in the same + // batch are the classic mislabel shape — fiction items where only the + // grounding tag wobbled. Demote them deterministically, no extra LLM call. + let contradictionDemotedCount = 0; + if (conversationRegister !== "real" && batchHasConstructedTag) { + for (let i = candidates.length - 1; i >= 0; i--) { + const candidate = candidates[i]; + if (DURABLE_CATEGORIES.has(candidate.category)) { + contradictionDemotedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: demoting real-tagged durable from ${conversationRegister}-register batch with constructed siblings (batch contradiction) category=${candidate.category} abstract=${JSON.stringify(candidate.abstract.slice(0, 120))}`); + candidates.splice(i, 1); + } + } } - this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}`); + this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`); return { status: "ok", candidates }; } // -------------------------------------------------------------------------- @@ -1199,6 +1246,12 @@ export class SmartExtractor { suppressed_until_turn: 0, memory_temporal_type: classifyTemporal(classifyText), valid_until: inferExpiry(classifyText), + // Grounding audit trail: which self-tag and batch register this + // memory was admitted under. Absent on legacy payloads. + ...(candidate.grounding ? { grounding: candidate.grounding } : {}), + ...(candidate.conversationRegister + ? { conversation_register: candidate.conversationRegister } + : {}), ...(admissionAudit ? { admission_audit: JSON.stringify(admissionAudit) } : {}), })); return { From 0b42b6b6bedcb5f3db258cf078c4dc1498c9767d Mon Sep 17 00:00:00 2001 From: Gorkem Date: Thu, 16 Jul 2026 09:19:29 +0300 Subject: [PATCH 08/11] test(extraction): cover the missing-register + constructed-sibling compound path Complements the existing "missing register, no constructed tags" test: a missing/malformed conversation_register defaults to "mixed", and a genuinely constructed sibling in the same batch is enough to arm the same batch-wide durable wipe a "fiction"/"mixed" register would. No production behavior change, just closes a gap the existing per-register tests didn't individually exercise. Co-Authored-By: Claude Sonnet 5 --- test/extraction-grounding-register.test.mjs | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/extraction-grounding-register.test.mjs b/test/extraction-grounding-register.test.mjs index 88e025309..99fc7b9bb 100644 --- a/test/extraction-grounding-register.test.mjs +++ b/test/extraction-grounding-register.test.mjs @@ -429,6 +429,28 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { assert.equal(stats.created, 2, "legacy payloads (no register, no constructed tags) must be unaffected"); }); + it("missing register (legacy payload) WITH a constructed sibling still arms the batch-wide durable wipe", async () => { + // Complements the test above: a missing/malformed register defaults to "mixed" (not + // "real"), and — unlike the no-constructed-tags case — a genuinely constructed sibling + // in the same batch is enough to arm the same wipe a "fiction"/"mixed" register would. + // This is the compound path a malformed extraction response could realistically hit + // (a legacy or dropped conversation_register field alongside a real constructed note), + // which the existing per-register tests don't individually cover. + const store = makeStore(); + const llm = makeLlm([...FACTUAL_CANDIDATES, MISLABELED_FICTION_CANDIDATES[2]]); // no conversation_register field + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const categories = persistedCategories(store); + assert.deepEqual( + categories, + ["events"], + "a constructed sibling arms the wipe even when conversation_register is missing entirely", + ); + assert.equal(stats.created, 1); + }); + it("propagates grounding and conversation_register into stored metadata", async () => { const store = makeStore(); const llm = makeLlm( From c6222900a13d1bcac2e7ea89fde2a86026d7810d Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 18 Jul 2026 20:05:13 +0300 Subject: [PATCH 09/11] fix(extraction): redefine grounding as about/within-the-fiction (v3) Fleet-proven refinement of the grounding model: 'real' includes an assertion ABOUT a fiction/game session (that it happened); 'constructed' is a claim true only WITHIN the fiction and is never stored, in any category or register. The batch register deterministically overrides per-item tag wobble for durable categories, and the admission judge's grounding rule is restated in candidate-content terms (the judge is transcript-free by design). --- dist/src/admission-control.js | 22 +-- dist/src/extraction-prompts.js | 24 ++-- dist/src/memory-categories.js | 7 +- dist/src/smart-extractor.js | 54 +++----- src/admission-control.ts | 24 ++-- src/extraction-prompts.ts | 24 ++-- src/memory-categories.ts | 7 +- src/smart-extractor.ts | 68 ++++----- test/extraction-grounding-register.test.mjs | 145 +++++++++++++------- 9 files changed, 203 insertions(+), 172 deletions(-) diff --git a/dist/src/admission-control.js b/dist/src/admission-control.js index 6371a780a..cee4e5aad 100644 --- a/dist/src/admission-control.js +++ b/dist/src/admission-control.js @@ -334,7 +334,7 @@ Use higher scores for durable profile facts, preferences, entity state, patterns Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. -Grounding rule: content asserted only inside roleplay, a game, fiction, a hypothetical, or a simulation is not a fact about the real user. If the excerpt shows a constructed frame (game rules, personas, "let's play", "suppose", canon of an invented world) and the candidate's claim lives inside that frame, score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity may still score moderately. +Grounding rule: this candidate's own grounding tag already passed the deterministic pre-admission check (a "constructed" tag is rejected before this scoring ever runs), but a mistagged or legacy candidate can still describe a claim that is true only WITHIN a fiction — a persona's invented trait from roleplay, a game's rules or score, drafted fiction, a hypothetical, or sample data. If the candidate's own content shows such a constructed frame and its claim lives inside it rather than being a claim ABOUT the fiction (e.g. that a session/game happened), score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity is a claim ABOUT the fiction and may still score moderately. Return JSON only: { @@ -461,7 +461,7 @@ export class AdmissionController { this.config = config; this.debugLog = debugLog; } - rejectConstructedDurable(candidate, now) { + rejectConstructed(candidate, now) { const featureScores = { utility: 0, confidence: 0, @@ -469,7 +469,7 @@ export class AdmissionController { recency: 0, typePrior: 0, }; - const reason = `Admission rejected (constructed-grounding candidate targeting durable category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const reason = `Admission rejected (constructed-grounding candidate category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; const audit = { version: "amac-v1", decision: "reject", @@ -488,7 +488,7 @@ export class AdmissionController { grounding: candidate.grounding, conversation_register: candidate.conversationRegister, }; - this.debugLog(`memory-lancedb-pro: admission-control: decision=reject (constructed durable short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`); + this.debugLog(`memory-lancedb-pro: admission-control: decision=reject (constructed short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`); return { decision: "reject", audit }; } async loadRelevantMatches(candidate, candidateVector, scopeFilter) { @@ -508,13 +508,13 @@ export class AdmissionController { async evaluate(params) { const now = params.now ?? Date.now(); // Deterministic pre-admission short-circuit: a candidate tagged - // "constructed" must never occupy a durable register, no matter how any - // downstream score would blend. Reject before any LLM call. Candidates - // without grounding metadata (legacy payloads) fall through to normal - // scoring. - if (params.candidate.grounding === "constructed" && - DURABLE_CATEGORIES.has(params.candidate.category)) { - return this.rejectConstructedDurable(params.candidate, now); + // "constructed" is true only within the fiction, never about the real + // world — it must never be stored, in any category or register, no + // matter how any downstream score would blend. Reject before any LLM + // call. Candidates without grounding metadata (legacy payloads) fall + // through to normal scoring. + if (params.candidate.grounding === "constructed") { + return this.rejectConstructed(params.candidate, now); } const relevantMatches = await this.loadRelevantMatches(params.candidate, params.candidateVector, params.scopeFilter); const utility = await scoreUtility(this.llm, this.config.utilityMode, params.candidate, params.conversationText); diff --git a/dist/src/extraction-prompts.js b/dist/src/extraction-prompts.js index 8329f09ed..132f6138e 100644 --- a/dist/src/extraction-prompts.js +++ b/dist/src/extraction-prompts.js @@ -31,7 +31,7 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile fact, preference, entity state, event, case, or pattern from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile detail, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. - Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. @@ -82,19 +82,23 @@ Set the top-level "conversation_register" field: ## Per-item grounding +Grounding describes the truth-grounding of the ASSERTION ITSELF, not which register the conversation happened in. Ask: is this claim true about the real world, or true only inside the fiction? + Tag every memory's "grounding" field: | grounding | Meaning | |-----------|---------| -| "real" | An assertion about the actual user, the real world, or this real working session — including a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | -| "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | +| "real" | The assertion is about the real world — INCLUDING an assertion ABOUT the fiction (e.g. "user and assistant played a one-round roleplay game where user was Admiral Vex" is a true statement about a real session, even though it describes fictional play). Also covers a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | The assertion holds only WITHIN the fiction — true in-character, in-game, or in-story, but not a fact about the real user or the real world (e.g. "user's favorite drink is plasma coffee", a persona's invented backstory, a game's score or rules) | + +One-line rule: **about-the-fiction is real; within-the-fiction is constructed.** Rules: - Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. -- If the conversation is in-character or mid-fiction, EVERY item derived from the narrative is "constructed" — an all-constructed batch is the normal result for such input. -- Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. From a constructed frame, the only extractable memory is a session-scoped "events" note that the real participants did the activity; the storage layer keeps at most one such note per extraction. That cap is a storage rule applied after tagging — it is NOT a tagging quota, so never re-tag extra items "real" to fit under it. +- A session-scoped "events" note that the real participants engaged in a game, roleplay, or other fiction is a REAL assertion — it is a true statement about what happened in the real session, not a claim that lives inside the fiction. Extract it like any other events item, under its natural category, with grounding "real". +- Do NOT lift any in-character proposition — an invented rule, a score, a bet, a persona's claim, a fictional preference or trait — into profile, preferences, entities, cases, or patterns. Such claims are true only within the fiction; if you extract one at all, tag it "constructed" so it is never mistaken for a fact about the real user. - A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. -- Self-consistency check before answering: items derived from the same fictional frame must share the same grounding tag. If your draft tags one game-derived item "constructed" and another game-derived item "real", re-check the batch — either the item is a genuine out-of-character aside, or its tag is wrong. +- Self-consistency check before answering: an item asserting what happened in the real session (including a session that was itself a game) is "real"; an item asserting a claim that is only true inside the story/game/persona is "constructed". If your draft tags the session-summary note itself as "constructed", re-check — it almost certainly describes a real event and should be "real". - If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. # Three-Level Structure @@ -152,7 +156,7 @@ Each example is a full output batch, because register and grounding are judged t } \`\`\` -## Mid-game conversation (register "fiction", ALL items constructed) +## Mid-game conversation (register "fiction", session note is real; canon is not extracted) Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { @@ -163,12 +167,12 @@ Input was one round of an in-character guessing game where a persona claimed to "abstract": "agent-one and agent-two ran a two-round puzzle exercise", "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", - "grounding": "constructed" + "grounding": "real" } ] } \`\`\` -Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. The whole batch carries "constructed" tags; that is the expected shape for mid-fiction input. +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. This session note is a true statement about a real session (the session happened), so it carries grounding "real" even though the batch register is "fiction" — about-the-fiction is real. ## Game with a genuine out-of-character aside (register "mixed") \`\`\`json @@ -187,7 +191,7 @@ Note: the persona's home, the invented drink, the house rule, and the bet are NO "abstract": "User and assistant played a riddle game", "overview": "## What happened\\n- One riddle game session", "content": "User and assistant played a short riddle game this session.", - "grounding": "constructed" + "grounding": "real" } ] } diff --git a/dist/src/memory-categories.js b/dist/src/memory-categories.js index edc745bef..75f7e96ff 100644 --- a/dist/src/memory-categories.js +++ b/dist/src/memory-categories.js @@ -58,7 +58,12 @@ export const APPEND_ONLY_CATEGORIES = new Set([ "events", "cases", ]); -/** Durable registers: constructed content must never land in these. */ +/** + * Durable categories: governs fiction-register batch enforcement (an + * in-fiction batch can never produce durable memories) and the batch + * contradiction check. Per-item grounding "constructed" is dropped + * unconditionally in every category, so this set does not gate that rule. + */ export const DURABLE_CATEGORIES = new Set([ "profile", "preferences", diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index b96e624db..2b80ce9af 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -582,7 +582,6 @@ export class SmartExtractor { let policyDroppedCount = 0; let constructedDroppedCount = 0; let fictionRegisterDroppedCount = 0; - let constructedEpisodicUsed = false; let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { @@ -617,46 +616,33 @@ export class SmartExtractor { this.debugLog(`memory-lancedb-pro: smart-extractor: dropping candidate due to episodic-only extraction policy category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); continue; } - // Option A: grounding-aware register filter. Missing/non-string/unrecognized + // Option A / v3: grounding-aware filter. Missing/non-string/unrecognized // per-item values fail open to "real" so a model that ignores the field - // can't break extraction. Constructed content (games, roleplay, fiction, - // hypotheticals, sample data) never becomes a durable profile/preferences/ - // entities/cases/patterns memory; at most one constructed "events" note per - // extraction records that the real participants did the activity. + // can't break extraction. Grounding describes the truth-grounding of the + // ASSERTION itself: "real" includes an assertion ABOUT a fiction/game + // session (e.g. that it happened); "constructed" is a claim true only + // WITHIN the fiction. A constructed-tagged candidate is never stored, + // in any category or register — there is no per-extraction cap anymore. const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; const grounding = rawGrounding === "constructed" ? "constructed" : "real"; if (rawGrounding === "constructed") { batchHasConstructedTag = true; } - if (conversationRegister === "fiction") { - // Register enforcement: an in-fiction batch can never produce durable - // memories, whatever the per-item self-tags claim (the per-item tags - // are exactly the wobble the batch register exists to override), and - // keeps at most one session-scoped events note. - if (DURABLE_CATEGORIES.has(category)) { - fictionRegisterDroppedCount++; - this.debugLog(`memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`); - continue; - } - if (constructedEpisodicUsed) { - fictionRegisterDroppedCount++; - this.debugLog(`memory-lancedb-pro: smart-extractor: dropping extra events candidate from fiction-register batch (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`); - continue; - } - constructedEpisodicUsed = true; + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override). + if (conversationRegister === "fiction" && DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; } - else if (grounding === "constructed") { - if (category !== "events") { - constructedDroppedCount++; - this.debugLog(`memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate outside events category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); - continue; - } - if (constructedEpisodicUsed) { - constructedDroppedCount++; - this.debugLog(`memory-lancedb-pro: smart-extractor: dropping extra constructed-grounding events candidate (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`); - continue; - } - constructedEpisodicUsed = true; + // Grounding enforcement: a constructed assertion is true only within + // the fiction, never about the real world — never stored, regardless + // of category or register. + if (grounding === "constructed") { + constructedDroppedCount++; + this.debugLog(`memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`); + continue; } candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); } diff --git a/src/admission-control.ts b/src/admission-control.ts index 8cec91cde..3e7ea97e1 100644 --- a/src/admission-control.ts +++ b/src/admission-control.ts @@ -489,7 +489,7 @@ Use higher scores for durable profile facts, preferences, entity state, patterns Use moderate scores for events worth an episodic record. Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. -Grounding rule: content asserted only inside roleplay, a game, fiction, a hypothetical, or a simulation is not a fact about the real user. If the excerpt shows a constructed frame (game rules, personas, "let's play", "suppose", canon of an invented world) and the candidate's claim lives inside that frame, score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity may still score moderately. +Grounding rule: this candidate's own grounding tag already passed the deterministic pre-admission check (a "constructed" tag is rejected before this scoring ever runs), but a mistagged or legacy candidate can still describe a claim that is true only WITHIN a fiction — a persona's invented trait from roleplay, a game's rules or score, drafted fiction, a hypothetical, or sample data. If the candidate's own content shows such a constructed frame and its claim lives inside it rather than being a claim ABOUT the fiction (e.g. that a session/game happened), score it near zero for the durable categories (profile, preferences, entities, cases, patterns) regardless of how well-formed it looks. A session-scoped events note that the participants did the activity is a claim ABOUT the fiction and may still score moderately. Return JSON only: { @@ -668,7 +668,7 @@ export class AdmissionController { private readonly debugLog: (msg: string) => void = () => {}, ) {} - private rejectConstructedDurable( + private rejectConstructed( candidate: CandidateMemory, now: number, ): AdmissionEvaluation { @@ -679,7 +679,7 @@ export class AdmissionController { recency: 0, typePrior: 0, }; - const reason = `Admission rejected (constructed-grounding candidate targeting durable category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; + const reason = `Admission rejected (constructed-grounding candidate category "${candidate.category}"; deterministic pre-admission short-circuit, no LLM call).`; const audit: AdmissionAuditRecord = { version: "amac-v1", decision: "reject", @@ -699,7 +699,7 @@ export class AdmissionController { conversation_register: candidate.conversationRegister, }; this.debugLog( - `memory-lancedb-pro: admission-control: decision=reject (constructed durable short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`, + `memory-lancedb-pro: admission-control: decision=reject (constructed short-circuit) candidate=${JSON.stringify(candidate.abstract.slice(0, 80))}`, ); return { decision: "reject", audit }; } @@ -742,15 +742,13 @@ export class AdmissionController { const now = params.now ?? Date.now(); // Deterministic pre-admission short-circuit: a candidate tagged - // "constructed" must never occupy a durable register, no matter how any - // downstream score would blend. Reject before any LLM call. Candidates - // without grounding metadata (legacy payloads) fall through to normal - // scoring. - if ( - params.candidate.grounding === "constructed" && - DURABLE_CATEGORIES.has(params.candidate.category) - ) { - return this.rejectConstructedDurable(params.candidate, now); + // "constructed" is true only within the fiction, never about the real + // world — it must never be stored, in any category or register, no + // matter how any downstream score would blend. Reject before any LLM + // call. Candidates without grounding metadata (legacy payloads) fall + // through to normal scoring. + if (params.candidate.grounding === "constructed") { + return this.rejectConstructed(params.candidate, now); } const relevantMatches = await this.loadRelevantMatches( diff --git a/src/extraction-prompts.ts b/src/extraction-prompts.ts index c18013839..85962c55d 100644 --- a/src/extraction-prompts.ts +++ b/src/extraction-prompts.ts @@ -35,7 +35,7 @@ ${conversationText} - Runtime scaffolding or orchestration wrappers such as "[Subagent Context]", "[Subagent Task]", bootstrap wrappers, task envelopes, or agent instructions — these are execution metadata, NEVER store them as memories - Recall queries / meta-questions: "Do you remember X?", "你还记得X吗?", "你知道我喜欢什么吗" — these are retrieval requests, NOT new information to store - Degraded or incomplete references: If the user mentions something vaguely ("that thing I said"), do NOT invent details or create a hollow memory -- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile fact, preference, entity state, event, case, or pattern from them or skip. +- Raw conversation carryover: quoted or attributed transcript blocks, especially 3+ lines of speaker text, are not memories by themselves. Distill a concrete profile detail, preference, entity state, event, case, or pattern from them or skip. - System/runtime artifacts: content containing "System:", compaction notices, model-switch/session-reset traces, tool-call transcripts, raw JSON blobs, or similar internal execution traces must be rejected unless a clean user fact can be extracted. - Fragment blobs: mixed filename shards, code snippets, metadata fields, or partial sentences that look like unprocessed context fragments should be skipped rather than preserved. - Atomic memory shape: each stored memory must read like one atomic profile fact, preference, entity state, event, case, or pattern. If a candidate reads like an excerpt, log, or raw transcript, compress it into one atomic statement or skip it. @@ -86,19 +86,23 @@ Set the top-level "conversation_register" field: ## Per-item grounding +Grounding describes the truth-grounding of the ASSERTION ITSELF, not which register the conversation happened in. Ask: is this claim true about the real world, or true only inside the fiction? + Tag every memory's "grounding" field: | grounding | Meaning | |-----------|---------| -| "real" | An assertion about the actual user, the real world, or this real working session — including a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | -| "constructed" | An assertion whose truth exists only inside an in-conversation constructed context: a game's rules/scores/bets, a role or persona's claims, drafted fiction, a hypothetical ("suppose X"), or sample/test data being manipulated | +| "real" | The assertion is about the real world — INCLUDING an assertion ABOUT the fiction (e.g. "user and assistant played a one-round roleplay game where user was Admiral Vex" is a true statement about a real session, even though it describes fictional play). Also covers a genuine first-person aside stated in passing during a game (e.g. "btw my flight is Tuesday") | +| "constructed" | The assertion holds only WITHIN the fiction — true in-character, in-game, or in-story, but not a fact about the real user or the real world (e.g. "user's favorite drink is plasma coffee", a persona's invented backstory, a game's score or rules) | + +One-line rule: **about-the-fiction is real; within-the-fiction is constructed.** Rules: - Grounding is judged PER ITEM, on that item's own content. There is no expected number of "real" or "constructed" tags per batch: a batch may be all-real, all-constructed, or anything between. -- If the conversation is in-character or mid-fiction, EVERY item derived from the narrative is "constructed" — an all-constructed batch is the normal result for such input. -- Do NOT lift any in-context proposition — an invented rule, a score, a bet, a persona's claim — into profile, preferences, entities, cases, or patterns. From a constructed frame, the only extractable memory is a session-scoped "events" note that the real participants did the activity; the storage layer keeps at most one such note per extraction. That cap is a storage rule applied after tagging — it is NOT a tagging quota, so never re-tag extra items "real" to fit under it. +- A session-scoped "events" note that the real participants engaged in a game, roleplay, or other fiction is a REAL assertion — it is a true statement about what happened in the real session, not a claim that lives inside the fiction. Extract it like any other events item, under its natural category, with grounding "real". +- Do NOT lift any in-character proposition — an invented rule, a score, a bet, a persona's claim, a fictional preference or trait — into profile, preferences, entities, cases, or patterns. Such claims are true only within the fiction; if you extract one at all, tag it "constructed" so it is never mistaken for a fact about the real user. - A real aside spoken during play is still "real" and should be extracted normally under its natural category, even though it occurred inside a constructed register. -- Self-consistency check before answering: items derived from the same fictional frame must share the same grounding tag. If your draft tags one game-derived item "constructed" and another game-derived item "real", re-check the batch — either the item is a genuine out-of-character aside, or its tag is wrong. +- Self-consistency check before answering: an item asserting what happened in the real session (including a session that was itself a game) is "real"; an item asserting a claim that is only true inside the story/game/persona is "constructed". If your draft tags the session-summary note itself as "constructed", re-check — it almost certainly describes a real event and should be "real". - If you are genuinely unsure about a single item, default to "real" — under-tagging as constructed risks losing a genuine fact. # Three-Level Structure @@ -156,7 +160,7 @@ Each example is a full output batch, because register and grounding are judged t } \`\`\` -## Mid-game conversation (register "fiction", ALL items constructed) +## Mid-game conversation (register "fiction", session note is real; canon is not extracted) Input was one round of an in-character guessing game where a persona claimed to live on a moon base and named an invented drink. \`\`\`json { @@ -167,12 +171,12 @@ Input was one round of an in-character guessing game where a persona claimed to "abstract": "agent-one and agent-two ran a two-round puzzle exercise", "overview": "## What happened\\n- Two agents played a puzzle guessing game with invented rules and a bet", "content": "agent-one and agent-two ran a two-round puzzle guessing exercise. The house rules, scores, and bet are part of the game, not durable facts.", - "grounding": "constructed" + "grounding": "real" } ] } \`\`\` -Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. The whole batch carries "constructed" tags; that is the expected shape for mid-fiction input. +Note: the persona's home, the invented drink, the house rule, and the bet are NOT extracted at all — not as profile, not as preferences, not as entities. This session note is a true statement about a real session (the session happened), so it carries grounding "real" even though the batch register is "fiction" — about-the-fiction is real. ## Game with a genuine out-of-character aside (register "mixed") \`\`\`json @@ -191,7 +195,7 @@ Note: the persona's home, the invented drink, the house rule, and the bet are NO "abstract": "User and assistant played a riddle game", "overview": "## What happened\\n- One riddle game session", "content": "User and assistant played a short riddle game this session.", - "grounding": "constructed" + "grounding": "real" } ] } diff --git a/src/memory-categories.ts b/src/memory-categories.ts index ca5fdf243..9158d3c33 100644 --- a/src/memory-categories.ts +++ b/src/memory-categories.ts @@ -91,7 +91,12 @@ export type CandidateGrounding = "real" | "constructed"; */ export type ConversationRegister = "real" | "mixed" | "fiction"; -/** Durable registers: constructed content must never land in these. */ +/** + * Durable categories: governs fiction-register batch enforcement (an + * in-fiction batch can never produce durable memories) and the batch + * contradiction check. Per-item grounding "constructed" is dropped + * unconditionally in every category, so this set does not gate that rule. + */ export const DURABLE_CATEGORIES = new Set([ "profile", "preferences", diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index d86c4313b..634c52f28 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -873,7 +873,6 @@ export class SmartExtractor { let policyDroppedCount = 0; let constructedDroppedCount = 0; let fictionRegisterDroppedCount = 0; - let constructedEpisodicUsed = false; let batchHasConstructedTag = false; for (const raw of result.memories) { if (!raw || typeof raw !== "object") { @@ -922,12 +921,13 @@ export class SmartExtractor { continue; } - // Option A: grounding-aware register filter. Missing/non-string/unrecognized + // Option A / v3: grounding-aware filter. Missing/non-string/unrecognized // per-item values fail open to "real" so a model that ignores the field - // can't break extraction. Constructed content (games, roleplay, fiction, - // hypotheticals, sample data) never becomes a durable profile/preferences/ - // entities/cases/patterns memory; at most one constructed "events" note per - // extraction records that the real participants did the activity. + // can't break extraction. Grounding describes the truth-grounding of the + // ASSERTION itself: "real" includes an assertion ABOUT a fiction/game + // session (e.g. that it happened); "constructed" is a claim true only + // WITHIN the fiction. A constructed-tagged candidate is never stored, + // in any category or register — there is no per-extraction cap anymore. const rawGrounding = typeof raw.grounding === "string" ? raw.grounding.toLowerCase().trim() : ""; const grounding: CandidateGrounding = @@ -936,42 +936,26 @@ export class SmartExtractor { batchHasConstructedTag = true; } - if (conversationRegister === "fiction") { - // Register enforcement: an in-fiction batch can never produce durable - // memories, whatever the per-item self-tags claim (the per-item tags - // are exactly the wobble the batch register exists to override), and - // keeps at most one session-scoped events note. - if (DURABLE_CATEGORIES.has(category)) { - fictionRegisterDroppedCount++; - this.debugLog( - `memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`, - ); - continue; - } - if (constructedEpisodicUsed) { - fictionRegisterDroppedCount++; - this.debugLog( - `memory-lancedb-pro: smart-extractor: dropping extra events candidate from fiction-register batch (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`, - ); - continue; - } - constructedEpisodicUsed = true; - } else if (grounding === "constructed") { - if (category !== "events") { - constructedDroppedCount++; - this.debugLog( - `memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate outside events category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, - ); - continue; - } - if (constructedEpisodicUsed) { - constructedDroppedCount++; - this.debugLog( - `memory-lancedb-pro: smart-extractor: dropping extra constructed-grounding events candidate (cap 1 per extraction) abstract=${JSON.stringify(abstract.slice(0, 120))}`, - ); - continue; - } - constructedEpisodicUsed = true; + // Register enforcement: an in-fiction batch can never produce durable + // memories, whatever the per-item self-tags claim (the per-item tags + // are exactly the wobble the batch register exists to override). + if (conversationRegister === "fiction" && DURABLE_CATEGORIES.has(category)) { + fictionRegisterDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping durable candidate from fiction-register batch category=${category} grounding=${grounding} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; + } + + // Grounding enforcement: a constructed assertion is true only within + // the fiction, never about the real world — never stored, regardless + // of category or register. + if (grounding === "constructed") { + constructedDroppedCount++; + this.debugLog( + `memory-lancedb-pro: smart-extractor: dropping constructed-grounding candidate category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`, + ); + continue; } candidates.push({ category, abstract, overview, content, grounding, conversationRegister }); diff --git a/test/extraction-grounding-register.test.mjs b/test/extraction-grounding-register.test.mjs index 99fc7b9bb..23bab1da7 100644 --- a/test/extraction-grounding-register.test.mjs +++ b/test/extraction-grounding-register.test.mjs @@ -1,13 +1,15 @@ /** - * Regression tests for grounding-aware extraction (Option A) and the - * scope-glob extraction-policy knob (Option C). + * Regression tests for grounding-aware extraction (Option A, v3 semantics) + * and the scope-glob extraction-policy knob (Option C). * * extractCandidates() tags each raw candidate "grounding": "real" | "constructed". - * A deterministic post-filter drops constructed candidates typed as - * profile/preferences/entities/cases/patterns, and allows at most one - * session-scoped "events" note per extraction for a constructed register - * (game/roleplay/fiction/hypothetical). A genuine first-person aside spoken - * during play is tagged "real" and must survive under its natural category. + * v3 grounding is about the truth-grounding of the ASSERTION itself, not which + * register the conversation happened in: "real" includes an assertion ABOUT a + * fiction/game session (e.g. that the session happened); "constructed" is a + * claim true only WITHIN the fiction (game canon, a persona's invented traits). + * A constructed-tagged candidate is NEVER stored, in any category or register — + * there is no per-extraction cap. A genuine first-person aside spoken during + * play is tagged "real" and must survive under its natural category. * * Fixtures are entirely synthetic (agent-one/agent-two, invented game * content) — no real fleet data. @@ -128,24 +130,33 @@ const GAME_CANDIDATES = [ abstract: "House rule: puzzle answers must be even numbers", overview: "## Rule\n- Even numbers only", content: "agent-two's puzzle house rule is that answers must always be even numbers.", - grounding: "constructed", + grounding: "constructed", // within-the-fiction: true only inside the game's rules }, { category: "cases", abstract: "Puzzle round 2 answer was 8", overview: "## Answer\n- 8", content: "The round 2 puzzle answer was 8, so agent-two won the bet.", - grounding: "constructed", + grounding: "constructed", // within-the-fiction: true only inside the game's outcome }, { category: "events", abstract: "agent-one and agent-two ran a two-round puzzle exercise", overview: "## What happened\n- Two agents played a puzzle guessing game", content: "agent-one and agent-two ran a two-round puzzle guessing exercise with an invented house rule and a bet.", - grounding: "constructed", + grounding: "real", // about-the-fiction: a true statement that the session happened }, ]; +/** A within-the-fiction plot beat — true only inside the story, not a real occurrence. */ +const CONSTRUCTED_PLOT_EVENT = { + category: "events", + abstract: "Admiral Vex's ship was boarded by pirates", + overview: "## In-story plot beat\n- Ship boarded mid-roleplay", + content: "In the roleplay, Admiral Vex's ship was boarded by pirates.", + grounding: "constructed", +}; + const REAL_ASIDE_CANDIDATE = { category: "events", abstract: "operators new laptop arrives Thursday", @@ -175,8 +186,8 @@ const FACTUAL_CANDIDATES = [ // Option A — grounding-aware extraction // ============================================================================ -describe("SmartExtractor grounding-aware extraction (Option A)", () => { - it("drops constructed non-events candidates from a game transcript, keeping only the episodic events note", async () => { +describe("SmartExtractor grounding-aware extraction (Option A, v3)", () => { + it("drops constructed candidates from a game transcript, keeping only the real session-events note", async () => { const store = makeStore(); const llm = makeLlm(GAME_CANDIDATES); const extractor = makeExtractor(makeEmbedder(), llm, store); @@ -184,27 +195,27 @@ describe("SmartExtractor grounding-aware extraction (Option A)", () => { const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); const categories = persistedCategories(store); - assert.deepEqual(categories, ["events"], "only the single constructed-but-episodic events note should survive"); + assert.deepEqual(categories, ["events"], "only the real (about-the-fiction) session-events note should survive"); assert.equal(stats.created, 1); }); - it("caps constructed events notes at one per extraction", async () => { + it("drops a constructed-tagged events candidate too — no cap, unconditional drop regardless of category", async () => { const store = makeStore(); - const twoEventsNotes = [ - GAME_CANDIDATES[2], + const twoConstructedEventsNotes = [ + CONSTRUCTED_PLOT_EVENT, { - ...GAME_CANDIDATES[2], - abstract: "agent-one and agent-two also played a second puzzle round", + ...CONSTRUCTED_PLOT_EVENT, + abstract: "Admiral Vex also repelled a second boarding attempt", }, ]; - const llm = makeLlm(twoEventsNotes); + const llm = makeLlm(twoConstructedEventsNotes); const extractor = makeExtractor(makeEmbedder(), llm, store); const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); const categories = persistedCategories(store); - assert.equal(categories.length, 1, "only the first constructed events note should survive the cap"); - assert.equal(stats.created, 1); + assert.equal(categories.length, 0, "constructed-tagged events candidates are dropped unconditionally in v3 — the old one-per-extraction cap no longer applies"); + assert.equal(stats.created, 0); }); it("keeps a genuine real first-person aside stated during play (false-positive guard)", async () => { @@ -219,12 +230,28 @@ describe("SmartExtractor grounding-aware extraction (Option A)", () => { abstracts.some((a) => a.includes("laptop arrives Thursday")), "the real aside must survive even though it was stated during a constructed register", ); - // The constructed episodic note + the real aside both land as "events". + // The real session-events note + the real aside both land as "events". const categories = persistedCategories(store); assert.equal(categories.filter((c) => c === "events").length, 2); assert.equal(categories.length, 2, "constructed non-events candidates must still be dropped"); }); + it("invariant: no stored row ever carries grounding 'constructed' — events included, no cap logic remains", async () => { + const store = makeStore(); + const llm = makeLlm( + [GAME_CANDIDATES[0], GAME_CANDIDATES[1], CONSTRUCTED_PLOT_EVENT, GAME_CANDIDATES[2]], + "mixed", + ); + const extractor = makeExtractor(makeEmbedder(), llm, store); + + const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); + + const metas = store.bulkStoreCalls.flat().map((e) => JSON.parse(e.metadata || "{}")); + assert.ok(metas.every((m) => m.grounding !== "constructed"), "no persisted row may carry grounding: constructed"); + assert.deepEqual(persistedCategories(store), ["events"], "only the real session-events note survives; every constructed-tagged candidate is dropped regardless of category"); + assert.equal(stats.created, 1); + }); + it("leaves a purely factual transcript unchanged (regression guard)", async () => { const store = makeStore(); const llm = makeLlm(FACTUAL_CANDIDATES); @@ -250,13 +277,15 @@ describe("SmartExtractor grounding-aware extraction (Option A)", () => { assert.equal(stats.created, 2, "non-string/unrecognized grounding values must fail open to 'real' and be kept"); }); - it("buildExtractionPrompt documents the grounding contract (structural check)", async () => { + it("buildExtractionPrompt documents the v3 grounding contract (structural check)", async () => { const { buildExtractionPrompt } = jiti("../src/extraction-prompts.ts"); const prompt = buildExtractionPrompt("some conversation", "test-user"); assert.match(prompt, /grounding/i); assert.match(prompt, /"real"\s*\|\s*"constructed"|real.*constructed/i); - assert.match(prompt, /at most one/i); + assert.match(prompt, /about-the-fiction is real/i, "the v3 about/within one-line rule must be present"); + assert.match(prompt, /within-the-fiction/i, "the within-the-fiction definition of constructed must be present"); + assert.doesNotMatch(prompt, /at most one/i, "the per-extraction constructed cap must be fully removed from the prompt"); }); }); @@ -331,21 +360,21 @@ const MISLABELED_FICTION_CANDIDATES = [ abstract: "User lives in Moon Base 9", overview: "## Background\n- Residence: Moon Base 9", content: "User lives in Moon Base 9.", - grounding: "real", // wrong: asserted only inside the game canon + grounding: "real", // wrong: within-the-fiction canon should be "constructed" }, { category: "preferences", abstract: "Favorite drink is nebula tea", overview: "## Preference\n- Drink: nebula tea", content: "User's favorite drink is nebula tea.", - grounding: "real", // wrong: asserted only inside the game canon + grounding: "real", // wrong: within-the-fiction canon should be "constructed" }, { category: "events", abstract: "User and assistant played one round of a space roleplay game", overview: "## What happened\n- One roleplay round", content: "User and assistant played one round of a space roleplay game this session.", - grounding: "constructed", + grounding: "real", // correct under v3: about-the-fiction, a true statement that the session happened }, ]; @@ -366,7 +395,7 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { assert.equal(stats.created, 1); }); - it("register 'fiction' caps events notes at one, regardless of per-item tags", async () => { + it("register 'fiction' no longer caps events notes — real session-events notes all survive (v3: cap removed)", async () => { const store = makeStore(); const llm = makeLlm( [ @@ -374,7 +403,6 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { { ...MISLABELED_FICTION_CANDIDATES[2], abstract: "User and assistant also played a bonus round", - grounding: "real", // tag wobble: still capped by the register rule }, ], "fiction", @@ -383,15 +411,15 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); - assert.equal(persistedCategories(store).length, 1); + assert.equal(persistedCategories(store).length, 2, "both real session-events notes survive; the v2 one-per-extraction cap no longer applies"); }); - it("register 'mixed' with a constructed sibling demotes real-tagged durables (batch contradiction check)", async () => { + it("register 'mixed' with a constructed sibling demotes real-tagged durables (batch contradiction check) and drops the constructed sibling itself", async () => { const store = makeStore(); const llm = makeLlm( [ - MISLABELED_FICTION_CANDIDATES[0], // profile tagged real - MISLABELED_FICTION_CANDIDATES[2], // events tagged constructed — the contradiction evidence + MISLABELED_FICTION_CANDIDATES[0], // profile tagged real (mislabeled) + CONSTRUCTED_PLOT_EVENT, // events tagged constructed — the contradiction evidence, and itself dropped unconditionally REAL_ASIDE_CANDIDATE, // real events aside must still survive ], "mixed", @@ -402,21 +430,21 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { const categories = persistedCategories(store); assert.ok(!categories.includes("profile"), "real-tagged durable must be demoted in a mixed-register batch with constructed siblings"); - assert.equal(categories.filter((c) => c === "events").length, 2, "both events notes (constructed episodic + real aside) survive"); + assert.deepEqual(categories, ["events"], "only the real aside survives: the constructed plot event is dropped unconditionally (v3), profile demoted by contradiction check"); }); - it("register 'real' fully trusts per-item tags (durables kept alongside a constructed events note)", async () => { + it("register 'real' fully trusts per-item tags for durables, but still drops a constructed-tagged item regardless of register", async () => { const store = makeStore(); const llm = makeLlm( - [FACTUAL_CANDIDATES[0], MISLABELED_FICTION_CANDIDATES[2]], + [FACTUAL_CANDIDATES[0], CONSTRUCTED_PLOT_EVENT], "real", ); const extractor = makeExtractor(makeEmbedder(), llm, store); await extractor.extractAndPersist("mostly factual conversation with a brief game", "s1"); - const categories = persistedCategories(store).sort(); - assert.deepEqual(categories, ["events", "preferences"], "register 'real' must not demote per-item-real durables"); + const categories = persistedCategories(store); + assert.deepEqual(categories, ["preferences"], "durable real-tagged preference survives; the constructed-tagged item is dropped even though the register is 'real'"); }); it("missing register (legacy payload) with no constructed tags behaves exactly as before", async () => { @@ -436,8 +464,19 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { // This is the compound path a malformed extraction response could realistically hit // (a legacy or dropped conversation_register field alongside a real constructed note), // which the existing per-register tests don't individually cover. + // + // MISLABELED_FICTION_CANDIDATES[2] no longer fits this fixture's role under v3: that + // abstract ("the session happened") is now correctly tagged "real" (about-the-fiction), + // so it can no longer arm the wipe. Seed an explicit constructed sibling instead + // (CONSTRUCTED_PLOT_EVENT, a within-the-fiction plot beat) alongside a genuine real + // events aside so something still survives to prove the durable-only wipe, not a + // blanket drop of everything. const store = makeStore(); - const llm = makeLlm([...FACTUAL_CANDIDATES, MISLABELED_FICTION_CANDIDATES[2]]); // no conversation_register field + const llm = makeLlm([ + FACTUAL_CANDIDATES[0], // preferences, durable — must be wiped by the constructed sibling below + CONSTRUCTED_PLOT_EVENT, // within-the-fiction: arms the wipe, and is itself dropped unconditionally + REAL_ASIDE_CANDIDATE, // real events aside must still survive + ]); // no conversation_register field const extractor = makeExtractor(makeEmbedder(), llm, store); const stats = await extractor.extractAndPersist(GAME_TRANSCRIPT, "s1"); @@ -466,7 +505,7 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { const eventsMeta = metas.find((m) => m.memory_category === "events"); assert.equal(prefMeta.grounding, "real"); assert.equal(prefMeta.conversation_register, "real"); - assert.equal(eventsMeta.grounding, "constructed"); + assert.equal(eventsMeta.grounding, "real"); assert.equal(eventsMeta.conversation_register, "real"); }); @@ -476,9 +515,8 @@ describe("SmartExtractor batch register signal (grounding v2)", () => { assert.match(prompt, /conversation_register/); assert.match(prompt, /"real\|mixed\|fiction"/); - assert.match(prompt, /storage rule applied after tagging/i, "the cap must be framed as a storage rule, not a tagging quota"); - assert.match(prompt, /all-constructed batch/i, "all-constructed batches must be documented as normal"); assert.match(prompt, /self-consistency/i, "the batch self-consistency instruction must be present"); + assert.doesNotMatch(prompt, /storage rule applied after tagging/i, "the deleted per-extraction cap language must not remain in the prompt"); }); }); @@ -529,16 +567,16 @@ describe("AdmissionController grounding awareness (grounding v2)", () => { assert.equal(evaluation.audit.conversation_register, "fiction"); }); - it("constructed events candidates still go through normal scoring (only durables short-circuit)", async () => { + it("v3: short-circuits constructed candidates of ANY category to reject with zero LLM calls (total enforcement)", async () => { const llm = makeAdmissionLlm(); const controller = new AdmissionController(admissionStore, llm, balanced); const evaluation = await controller.evaluate({ candidate: { category: "events", - abstract: "User and assistant played one round of a space roleplay game", - overview: "## What happened\n- One roleplay round", - content: "User and assistant played one round of a space roleplay game this session.", + abstract: "Admiral Vex's ship was boarded by pirates", + overview: "## In-story plot beat\n- Ship boarded mid-roleplay", + content: "In the roleplay, Admiral Vex's ship was boarded by pirates.", grounding: "constructed", conversationRegister: "fiction", }, @@ -547,7 +585,9 @@ describe("AdmissionController grounding awareness (grounding v2)", () => { scopeFilter: ["global"], }); - assert.equal(llm.utilityCalls, 1, "events candidates use the normal scoring path"); + assert.equal(evaluation.decision, "reject"); + assert.equal(llm.utilityCalls, 0, "v3 total enforcement must short-circuit events too, not just durable categories"); + assert.match(evaluation.audit.reason, /constructed-grounding/); assert.equal(evaluation.audit.grounding, "constructed"); }); @@ -586,7 +626,7 @@ describe("AdmissionController grounding awareness (grounding v2)", () => { abstract: "User and assistant played one round of a space roleplay game", overview: "", content: "One roleplay round happened.", - grounding: "constructed", + grounding: "real", // v3: real candidates reach normal scoring; constructed short-circuits before this prompt is ever built conversationRegister: "fiction", }, candidateVector: [1, 0, 0], @@ -596,8 +636,13 @@ describe("AdmissionController grounding awareness (grounding v2)", () => { assert.equal(llm.prompts.length, 1); const prompt = llm.prompts[0]; - assert.match(prompt, /Grounding: constructed/); - assert.match(prompt, /Conversation register: fiction/); + // The judge does not see a per-candidate "Grounding:"/"Conversation + // register:" field (the candidate block is transcript-free and shared + // with every other admission path) -- grounding enforcement rides the + // deterministic constructed short-circuit (this candidate is "real", so + // it reaches normal scoring) plus this prose rule paragraph, both + // carried in the system prompt. + assert.match(prompt, /roleplay|fiction|hypothetical|simulation/i); for (const register of ["profile", "preferences", "entities", "events", "cases", "patterns"]) { assert.match(prompt, new RegExp(register), `utility prompt must name the ${register} register`); } From 54f278405962278f3994813799c1491a3b1b7b94 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 19 Jul 2026 05:59:01 +0300 Subject: [PATCH 10/11] chore(dist): rebuild on current master --- package.json | 2 +- scripts/ci-test-manifest.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 0baf407e7..c43770dc4 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/extraction-grounding-register.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/extraction-grounding-register.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", "test:llm-clients-and-auth": "node scripts/run-ci-tests.mjs --group llm-clients-and-auth", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 142b655e2..befa346f4 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -37,7 +37,6 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-noise-gating.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/extraction-grounding-register.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-capability-runtime.test.mjs" }, { group: "core-regression", runner: "node", file: "test/dreaming-engine.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-governance-tools.test.mjs", args: ["--test"] }, @@ -111,6 +110,7 @@ export const CI_TEST_MANIFEST = [ // Delete/delete-bulk must synchronously invalidate in-process reflection read caches { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/reflection-mapped-rows-admission.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/extraction-grounding-register.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { From 3a9648d4d59bb9c88a85fffa4e46e7a1d02b3745 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Mon, 20 Jul 2026 13:02:12 +0300 Subject: [PATCH 11/11] fix(extraction): stop training the noise bank on validation-emptied batches, log contradiction demotions extractAndPersist fed learnAsNoise(conversationText) whenever extraction returned ok with zero candidates: the model found nothing, strongest noise signal. But a batch can reach zero candidates through validation drops and the batch contradiction demotion, not genuine emptiness. In that shape the model DID find candidates; feeding the conversation to the embedding noise bank teaches that content like this is noise and risks pre-filtering similar real content away from future extractions. Gate noise learning on the RAW model output being empty (rawCandidateCount now rides the ok result), and log the skip at the standard level. Also surface the batch contradiction check at the standard log level (one line with count and register) when it demotes anything: a fully demoted batch was previously indistinguishable from a model that found nothing, without debug logging. --- dist/src/smart-extractor.js | 17 +++++- src/smart-extractor.ts | 28 +++++++-- test/smart-extractor-noise-gating.test.mjs | 70 +++++++++++++++++++--- 3 files changed, 102 insertions(+), 13 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 2b80ce9af..0833c2b2a 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -278,10 +278,17 @@ export class SmartExtractor { const candidates = extraction.candidates; if (candidates.length === 0) { this.log("memory-pro: smart-extractor: no memories extracted"); - if (extraction.status === "ok") { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); } + else if (extraction.status === "ok") { + // The model DID emit candidates; validation dropped or demoted them + // all. That is a policy verdict about those candidates, not evidence + // the conversation is noise — learning it as noise would pre-filter + // similar real content away from future extractions. + this.log(`memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`); + } else { this.debugLog(`memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`); } @@ -662,8 +669,14 @@ export class SmartExtractor { } } } + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log(`memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`); + } this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`); - return { status: "ok", candidates }; + return { status: "ok", candidates, rawCandidateCount: result.memories.length }; } // -------------------------------------------------------------------------- // Step 2: Dedup + Persist diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index 634c52f28..17223f1b8 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -67,9 +67,12 @@ type PendingSupersedeInvalidation = { // Discriminates WHY extractCandidates() came back empty, so the caller can // tell a genuine "LLM found nothing" verdict from a gateway/model failure or -// a malformed response shape — only the former is a real noise signal. +// a malformed response shape. rawCandidateCount carries how many items the +// model emitted BEFORE validation: only "ok" with a raw count of zero is a +// real noise signal — a validation-emptied batch is a policy verdict about +// candidates the model did find, not evidence the conversation is noise. type ExtractCandidatesResult = - | { status: "ok"; candidates: CandidateMemory[] } + | { status: "ok"; candidates: CandidateMemory[]; rawCandidateCount: number } | { status: "llm_failure"; candidates: [] } | { status: "malformed"; candidates: [] }; @@ -461,9 +464,17 @@ export class SmartExtractor { if (candidates.length === 0) { this.log("memory-pro: smart-extractor: no memories extracted"); - if (extraction.status === "ok") { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); + } else if (extraction.status === "ok") { + // The model DID emit candidates; validation dropped or demoted them + // all. That is a policy verdict about those candidates, not evidence + // the conversation is noise — learning it as noise would pre-filter + // similar real content away from future extractions. + this.log( + `memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`, + ); } else { this.debugLog( `memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`, @@ -980,11 +991,20 @@ export class SmartExtractor { } } + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log( + `memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`, + ); + } + this.debugLog( `memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`, ); - return { status: "ok", candidates }; + return { status: "ok", candidates, rawCandidateCount: result.memories.length }; } // -------------------------------------------------------------------------- diff --git a/test/smart-extractor-noise-gating.test.mjs b/test/smart-extractor-noise-gating.test.mjs index 2382fca96..0a1816246 100644 --- a/test/smart-extractor-noise-gating.test.mjs +++ b/test/smart-extractor-noise-gating.test.mjs @@ -4,10 +4,12 @@ * extractCandidates() can come back with zero candidates for four different * reasons: the LLM/gateway call failed outright, the response had an * unexpected shape, the LLM genuinely returned an empty memories list, or - * every parsed candidate was dropped by local validation. Only the latter - * two are a real "nothing to remember" signal — the first two are failures - * that must NOT be fed into the noise-prototype bank, or gateway outages - * silently poison future extraction/recall. + * every parsed candidate was dropped/demoted by local validation. ONLY the + * genuinely-empty list is a real "nothing to remember" signal. Failures must + * not train the bank (gateway outages would poison it), and a + * validation-emptied batch must not either: the model DID find candidates + * there, so teaching the bank "this conversation is noise" would pre-filter + * similar real content away from future extractions. */ import { describe, it } from "node:test"; @@ -42,6 +44,7 @@ function makeLlm(behavior) { if (behavior === "malformed_non_array") return { memories: "not-an-array" }; if (behavior === "empty") return { memories: [] }; if (Array.isArray(behavior)) return { memories: behavior }; + if (behavior && typeof behavior === "object" && behavior.raw) return behavior.raw; throw new Error(`unsupported test behavior: ${behavior}`); }, }; @@ -130,16 +133,69 @@ describe("SmartExtractor noise-bank learning gate", () => { assert.equal(noiseBank.learnCalls.length, 1, "a genuine empty list is a real noise signal"); }); - it("learns noise when every parsed candidate is dropped by local validation", async () => { + it("does NOT learn noise when validation drops every parsed candidate (policy verdict, not a noise signal)", async () => { const noiseBank = makeNoiseBank(); + const logs = []; // "hi" is a valid category but fails the length>=5 abstract check, so the - // parsed response is well-formed yet every candidate is filtered out. + // parsed response is well-formed yet every candidate is filtered out. The + // model DID emit a candidate, so the conversation must not train the bank. const llm = makeLlm([{ category: "preferences", abstract: "hi", overview: "", content: "" }]); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + assert.equal(noiseBank.learnCalls.length, 0, "a validation-emptied batch must not train the noise bank"); + assert.ok( + logs.some((msg) => msg.includes("skipping noise-bank learning (validation emptied the batch")), + "the skip must be visible at the standard log level", + ); + }); + + it("does NOT learn noise when the batch contradiction check demotes every candidate", async () => { + const noiseBank = makeNoiseBank(); + // Mixed-register batch with a constructed sibling: the real-tagged durable + // is demoted by the batch contradiction check, the constructed one is + // dropped by grounding enforcement. Raw output had 2 candidates, so the + // conversation (which contains real facts the policy demoted) must not be + // fed to the noise bank as a noise exemplar. + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank }); await extractor.extractAndPersist("some conversation text", "s1"); await flushMicrotasks(); - assert.equal(noiseBank.learnCalls.length, 1, "a validly-parsed-but-empty-after-filtering result is still a real noise signal"); + assert.equal(noiseBank.learnCalls.length, 0, "a demotion-emptied batch must not train the noise bank"); + }); + + it("logs the batch contradiction demotion at the standard log level, with count and register", async () => { + const noiseBank = makeNoiseBank(); + const logs = []; + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + const demotionLine = logs.find((msg) => msg.includes("batch contradiction demoted")); + assert.ok(demotionLine, "a fully-demoted batch must be distinguishable from 'model found nothing' without debug logging"); + assert.ok(demotionLine.includes("1"), "the demotion line must carry the demoted count"); + assert.ok(demotionLine.includes("mixed"), "the demotion line must carry the batch register"); }); });