diff --git a/cli.ts b/cli.ts index 31366f6d7..5e5075aaf 100644 --- a/cli.ts +++ b/cli.ts @@ -20,6 +20,8 @@ import type { MemoryScopeManager } from "./src/scopes.js"; import type { MemoryMigrator } from "./src/migrate.js"; import { createMemoryUpgrader } from "./src/memory-upgrader.js"; import type { LlmClient } from "./src/llm-client.js"; +import type { MdMirrorWriter } from "./src/tools.js"; +import { runConsolidate, formatConsolidateCostPreview, formatConsolidatePlanForDisplay, pluralCount } from "./src/consolidate.js"; import { getDefaultOauthModelForProvider, getOAuthProviderLabel, @@ -41,6 +43,7 @@ interface CLIContext { migrator: MemoryMigrator; embedder?: import("./src/embedder.js").Embedder; llmClient?: LlmClient; + mdMirror?: MdMirrorWriter | null; pluginId?: string; pluginConfig?: Record; // Called synchronously after a delete or delete-bulk command actually removes @@ -1306,6 +1309,7 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { .option("--limit ", "Maximum number of results", "20") .option("--offset ", "Number of results to skip", "0") .option("--json", "Output as JSON") + .option("--include-invalidated", "Include invalidated/superseded rows (excluded by default)", false) .action(async (options) => { try { const limit = parseInt(options.limit) || 20; @@ -1320,7 +1324,8 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { scopeFilter, options.category, limit, - offset + offset, + { excludeInactive: !options.includeInvalidated }, ); if (options.json) { @@ -1438,6 +1443,7 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { writeJson(summary); } else { console.log(`Memory Statistics:`); + console.log(`• Live memories: ${stats.liveCount}`); console.log(`• Total memories: ${stats.totalCount}`); console.log(`• Available scopes: ${scopeStats.totalScopes}`); console.log(`• Retrieval mode: ${retrievalConfig.mode}`); @@ -1548,10 +1554,15 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { scopeFilter = [options.scope]; } + // excludeInactive:false -- export is a backup/forensic view and must + // keep full-dump semantics, including invalidated/superseded rows + // (item 6, PR #946). const memories = await context.store.list( scopeFilter, options.category, - 1000 // Large limit for export + 1000, // Large limit for export + 0, + { excludeInactive: false }, ); const exportData = { @@ -1595,11 +1606,18 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { .option("--category ", "Export specific category") .option("--limit ", "Maximum memories to export", "1000") .option("--dry-run", "Show what would be written without creating files") + .option("--include-invalidated", "Include invalidated/superseded rows (excluded by default)", false) .action(async (options) => { try { const limit = clampInt(Number(options.limit), 1, 10000); const scopeFilter = options.scope ? [String(options.scope)] : undefined; - const memories = await context.store.list(scopeFilter, options.category, limit); + const memories = await context.store.list( + scopeFilter, + options.category, + limit, + 0, + { excludeInactive: !options.includeInvalidated }, + ); const vault = path.resolve(String(options.vault)); const root = path.join(vault, "00-AI-Memory"); @@ -2212,6 +2230,213 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { process.exit(1); } }); + + // consolidate: reconcile duplicate/contradictory rows already in the store + registerConsolidateCommand(memory, context); +} + +/** + * Item 7: the real confirm implementation wired to a real CLI invocation. + * Fails closed -- non-interactive (either stream not a TTY) resolves false + * without ever reading anything, so a scripted/piped invocation without + * --yes aborts cleanly instead of hanging on stdin or silently proceeding. + * Streams are injectable so tests can drive both branches deterministically. + */ +export function createConsolidateConfirm(streams?: { + stdin?: NodeJS.ReadableStream & { isTTY?: boolean }; + stdout?: NodeJS.WritableStream & { isTTY?: boolean }; +}): (promptText: string) => Promise { + const stdin = streams?.stdin ?? process.stdin; + const stdout = streams?.stdout ?? process.stdout; + + return async (promptText: string): Promise => { + if (!stdin.isTTY || !stdout.isTTY) { + return false; + } + const rl = readline.createInterface({ input: stdin as NodeJS.ReadableStream, output: stdout as NodeJS.WritableStream }); + try { + const answer = await new Promise((resolve) => rl.question(promptText, resolve)); + return answer.trim() === "YES"; + } finally { + rl.close(); + } + }; +} + +/** Item 8: renders the full plan (verdict, member ids, survivor, exact merge content) for user review before the apply prompt. */ +async function loadConsolidateSettledLedger(ledgerPath: string): Promise> { + try { + const raw = await readFile(ledgerPath, "utf-8"); + const parsed = JSON.parse(raw) as Record; + return parsed && typeof parsed === "object" ? parsed : {}; + } catch { + return {}; + } +} + +async function saveConsolidateSettledLedger( + ledgerPath: string, + ledger: Record, + scope: string, + newlySettled: string[], +): Promise { + if (newlySettled.length === 0) return; + try { + const merged = new Set([...(ledger[scope] ?? []), ...newlySettled]); + ledger[scope] = [...merged]; + await writeFile(ledgerPath, JSON.stringify(ledger, null, 2), "utf-8"); + } catch (err) { + console.warn(`consolidate: could not persist settled ledger: ${String(err)}`); + } +} + +function registerConsolidateCommand(memory: Command, context: CLIContext) { + memory + .command("consolidate") + .description("Reconcile duplicate or contradictory memories already in the store across write lanes (dry-run by default)") + .requiredOption("--agent ", "Agent whose memory to consolidate (scope agent:; journal-mirror writes route to this agent's workspace)") + .option("--category ", "Limit to one smart category (profile|preferences|entities|events|cases|patterns)") + .option("--since ", "Only consider rows stored at or after this ISO timestamp") + .option("--apply", "Apply the consolidation plan immediately (default is a dry-run preview with an interactive apply prompt)", false) + .option("--yes", "Skip the LLM-cost confirmation prompt (required for non-interactive/automated runs)", false) + .option("--include-reflection-slices", "Include reflection writer-2 slice rows in the scan (excluded by default)", false) + .action(async (options: { + category?: string; + since?: string; + apply: boolean; + yes: boolean; + includeReflectionSlices: boolean; + agent: string; + }) => { + try { + if (!context.llmClient) { + console.error("consolidate: no LLM client configured, cannot make consolidation decisions"); + process.exit(1); + } + if (!context.embedder) { + console.error("consolidate: no embedder configured, cannot re-embed merged rows"); + process.exit(1); + } + const llmClient = context.llmClient; + const embedder = context.embedder; + + let sinceMs: number | undefined; + if (options.since) { + const parsed = Date.parse(options.since); + if (Number.isNaN(parsed)) { + console.error(`consolidate: invalid --since timestamp "${options.since}"`); + process.exit(1); + } + sinceMs = parsed; + } + + const mdMirror = context.mdMirror; + const confirm = createConsolidateConfirm(); + const scope = `agent:${options.agent}`; + const settledLedgerPath = + typeof context.store.dbPath === "string" && context.store.dbPath.length > 0 + ? path.join(context.store.dbPath, "consolidate-settled.json") + : undefined; + const settledLedger = settledLedgerPath ? await loadConsolidateSettledLedger(settledLedgerPath) : {}; + + const result = await runConsolidate( + { + fetchRows: (scopeFilter, maxTimestamp, limit) => + context.store.fetchForCompaction(maxTimestamp, scopeFilter, limit), + update: (id, patch, scopeFilter) => context.store.update(id, patch, scopeFilter), + getById: (id, scopeFilter) => context.store.getById(id, scopeFilter), + embed: (text) => embedder.embedPassage(text), + completeJson: (prompt, label, system, temperature) => llmClient.completeJson(prompt, label, system, temperature), + log: (message) => console.warn(message), + confirmCost: async (message) => { + console.log(`\n${message}`); + return confirm("Proceed with these LLM calls? Type YES to continue: "); + }, + confirmApply: async (message, clusters) => { + console.log(`\n${formatConsolidatePlanForDisplay(clusters)}`); + console.log(`\n${message}`); + return confirm("Type YES to apply: "); + }, + onAudit: mdMirror + ? async (audit) => { + const summary = `${audit.action} survivor=${audit.survivorId.slice(0, 8)} absorbed=${audit.absorbedIds.map((id) => id.slice(0, 8)).join(",")} reason="${audit.reason}"`; + await mdMirror( + { text: summary, category: "consolidation", scope: audit.scope, timestamp: Date.now() }, + { source: `memory-consolidate:${audit.action}`, agentId: options.agent }, + ); + } + : undefined, + }, + { + scope, + category: options.category as import("./src/memory-categories.js").MemoryCategory | undefined, + sinceMs, + includeReflectionSlices: options.includeReflectionSlices, + apply: options.apply === true, + autoConfirm: options.yes === true, + settledFingerprints: new Set(settledLedger[scope] ?? []), + }, + ); + + if (result.status === "aborted") { + const declined = (result.abortReason ?? "").includes("cost gate declined"); + if (declined) { + console.log(`consolidate: cancelled at the cost gate — no LLM calls were made.`); + console.log(`Pass --yes to skip this prompt (e.g. for automation).`); + } else { + console.error(`consolidate: aborted -- ${result.abortReason}`); + if (result.costPreview) { + console.error(formatConsolidateCostPreview(result.costPreview)); + } + console.error(`Pass --yes to skip this prompt (e.g. for automation), or re-run interactively and type YES.`); + } + process.exit(1); + } + + console.log(`Scanned ${pluralCount(result.scanned, "row")}, ${result.eligible} eligible for consolidation.`); + const settledNote = result.settledSkipped > 0 ? ` (${result.settledSkipped} settled in previous runs)` : ""; + if (result.clusters.length === 0) { + console.log(`0 candidates${settledNote} — nothing to consolidate.\n`); + } else { + console.log(`Decided ${pluralCount(result.clusters.length, "cluster")}${settledNote}:\n`); + } + + for (const cluster of result.clusters) { + if (cluster.malformed) { + const label = cluster.failure === "call-failed" ? "undecided: LLM call failed" : "skipped: malformed verdict"; + console.log(` [${label}] ${pluralCount(cluster.memberIds.length, "row")}`); + for (const text of cluster.memberTexts) console.log(` - "${text}"`); + continue; + } + const blockedNote = cluster.blocked === "append-only-shield" ? " — BLOCKED by append-only shield (not applied)" : ""; + console.log(` [${cluster.verdict!.verdict}] ${pluralCount(cluster.memberIds.length, "row")} — ${cluster.verdict!.reason}${blockedNote}`); + for (const text of cluster.memberTexts) console.log(` - "${text}"`); + } + + if (result.staleSkipped.length > 0) { + console.log(`\n${pluralCount(result.staleSkipped.length, "cluster")} skipped: changed since the plan was built (stale).`); + } + + if (result.status === "completed" && settledLedgerPath) { + await saveConsolidateSettledLedger(settledLedgerPath, settledLedger, scope, result.newlySettled); + } + + if (!result.executed) { + if (!options.apply) { + console.log(`\nNo changes applied.`); + } + return; + } + + const failureNotes: string[] = []; + if (result.skippedMalformed > 0) failureNotes.push(`${pluralCount(result.skippedMalformed, "cluster")} skipped due to malformed verdicts`); + if (result.undecidedCallFailed > 0) failureNotes.push(`${pluralCount(result.undecidedCallFailed, "cluster")} undecided because the decide call failed`); + console.log(`\nApplied ${pluralCount(result.applied.length, "action")}${failureNotes.length ? "; " + failureNotes.join("; ") : ""}.`); + } catch (error) { + console.error("consolidate failed:", error); + process.exit(1); + } + }); } // ============================================================================ diff --git a/dist/cli.js b/dist/cli.js index 2bac4689d..9550cb0b9 100644 --- a/dist/cli.js +++ b/dist/cli.js @@ -11,6 +11,7 @@ import { loadLanceDB } from "./src/store.js"; import { parseSmartMetadata, buildSmartMetadata, stringifySmartMetadata, } from "./src/smart-metadata.js"; import { createRetriever } from "./src/retriever.js"; import { createMemoryUpgrader } from "./src/memory-upgrader.js"; +import { runConsolidate, formatConsolidateCostPreview, formatConsolidatePlanForDisplay, pluralCount } from "./src/consolidate.js"; import { getDefaultOauthModelForProvider, getOAuthProviderLabel, isOauthModelSupported, listOAuthProviders, normalizeOauthModel, normalizeOAuthProviderId, performOAuthLogin, } from "./src/llm-oauth.js"; // ============================================================================ // Utility Functions @@ -1044,6 +1045,7 @@ export function registerMemoryCLI(program, context) { .option("--limit ", "Maximum number of results", "20") .option("--offset ", "Number of results to skip", "0") .option("--json", "Output as JSON") + .option("--include-invalidated", "Include invalidated/superseded rows (excluded by default)", false) .action(async (options) => { try { const limit = parseInt(options.limit) || 20; @@ -1052,7 +1054,7 @@ export function registerMemoryCLI(program, context) { if (options.scope) { scopeFilter = [options.scope]; } - const memories = await context.store.list(scopeFilter, options.category, limit, offset); + const memories = await context.store.list(scopeFilter, options.category, limit, offset, { excludeInactive: !options.includeInvalidated }); if (options.json) { writeJson(memories); } @@ -1162,6 +1164,7 @@ export function registerMemoryCLI(program, context) { } else { console.log(`Memory Statistics:`); + console.log(`• Live memories: ${stats.liveCount}`); console.log(`• Total memories: ${stats.totalCount}`); console.log(`• Available scopes: ${scopeStats.totalScopes}`); console.log(`• Retrieval mode: ${retrievalConfig.mode}`); @@ -1266,8 +1269,11 @@ export function registerMemoryCLI(program, context) { if (options.scope) { scopeFilter = [options.scope]; } - const memories = await context.store.list(scopeFilter, options.category, 1000 // Large limit for export - ); + // excludeInactive:false -- export is a backup/forensic view and must + // keep full-dump semantics, including invalidated/superseded rows + // (item 6, PR #946). + const memories = await context.store.list(scopeFilter, options.category, 1000, // Large limit for export + 0, { excludeInactive: false }); const exportData = { version: "1.0", exportedAt: new Date().toISOString(), @@ -1307,11 +1313,12 @@ export function registerMemoryCLI(program, context) { .option("--category ", "Export specific category") .option("--limit ", "Maximum memories to export", "1000") .option("--dry-run", "Show what would be written without creating files") + .option("--include-invalidated", "Include invalidated/superseded rows (excluded by default)", false) .action(async (options) => { try { const limit = clampInt(Number(options.limit), 1, 10000); const scopeFilter = options.scope ? [String(options.scope)] : undefined; - const memories = await context.store.list(scopeFilter, options.category, limit); + const memories = await context.store.list(scopeFilter, options.category, limit, 0, { excludeInactive: !options.includeInvalidated }); const vault = path.resolve(String(options.vault)); const root = path.join(vault, "00-AI-Memory"); let created = 0; @@ -1843,6 +1850,185 @@ export function registerMemoryCLI(program, context) { process.exit(1); } }); + // consolidate: reconcile duplicate/contradictory rows already in the store + registerConsolidateCommand(memory, context); +} +/** + * Item 7: the real confirm implementation wired to a real CLI invocation. + * Fails closed -- non-interactive (either stream not a TTY) resolves false + * without ever reading anything, so a scripted/piped invocation without + * --yes aborts cleanly instead of hanging on stdin or silently proceeding. + * Streams are injectable so tests can drive both branches deterministically. + */ +export function createConsolidateConfirm(streams) { + const stdin = streams?.stdin ?? process.stdin; + const stdout = streams?.stdout ?? process.stdout; + return async (promptText) => { + if (!stdin.isTTY || !stdout.isTTY) { + return false; + } + const rl = readline.createInterface({ input: stdin, output: stdout }); + try { + const answer = await new Promise((resolve) => rl.question(promptText, resolve)); + return answer.trim() === "YES"; + } + finally { + rl.close(); + } + }; +} +/** Item 8: renders the full plan (verdict, member ids, survivor, exact merge content) for user review before the apply prompt. */ +async function loadConsolidateSettledLedger(ledgerPath) { + try { + const raw = await readFile(ledgerPath, "utf-8"); + const parsed = JSON.parse(raw); + return parsed && typeof parsed === "object" ? parsed : {}; + } + catch { + return {}; + } +} +async function saveConsolidateSettledLedger(ledgerPath, ledger, scope, newlySettled) { + if (newlySettled.length === 0) + return; + try { + const merged = new Set([...(ledger[scope] ?? []), ...newlySettled]); + ledger[scope] = [...merged]; + await writeFile(ledgerPath, JSON.stringify(ledger, null, 2), "utf-8"); + } + catch (err) { + console.warn(`consolidate: could not persist settled ledger: ${String(err)}`); + } +} +function registerConsolidateCommand(memory, context) { + memory + .command("consolidate") + .description("Reconcile duplicate or contradictory memories already in the store across write lanes (dry-run by default)") + .requiredOption("--agent ", "Agent whose memory to consolidate (scope agent:; journal-mirror writes route to this agent's workspace)") + .option("--category ", "Limit to one smart category (profile|preferences|entities|events|cases|patterns)") + .option("--since ", "Only consider rows stored at or after this ISO timestamp") + .option("--apply", "Apply the consolidation plan immediately (default is a dry-run preview with an interactive apply prompt)", false) + .option("--yes", "Skip the LLM-cost confirmation prompt (required for non-interactive/automated runs)", false) + .option("--include-reflection-slices", "Include reflection writer-2 slice rows in the scan (excluded by default)", false) + .action(async (options) => { + try { + if (!context.llmClient) { + console.error("consolidate: no LLM client configured, cannot make consolidation decisions"); + process.exit(1); + } + if (!context.embedder) { + console.error("consolidate: no embedder configured, cannot re-embed merged rows"); + process.exit(1); + } + const llmClient = context.llmClient; + const embedder = context.embedder; + let sinceMs; + if (options.since) { + const parsed = Date.parse(options.since); + if (Number.isNaN(parsed)) { + console.error(`consolidate: invalid --since timestamp "${options.since}"`); + process.exit(1); + } + sinceMs = parsed; + } + const mdMirror = context.mdMirror; + const confirm = createConsolidateConfirm(); + const scope = `agent:${options.agent}`; + const settledLedgerPath = typeof context.store.dbPath === "string" && context.store.dbPath.length > 0 + ? path.join(context.store.dbPath, "consolidate-settled.json") + : undefined; + const settledLedger = settledLedgerPath ? await loadConsolidateSettledLedger(settledLedgerPath) : {}; + const result = await runConsolidate({ + fetchRows: (scopeFilter, maxTimestamp, limit) => context.store.fetchForCompaction(maxTimestamp, scopeFilter, limit), + update: (id, patch, scopeFilter) => context.store.update(id, patch, scopeFilter), + getById: (id, scopeFilter) => context.store.getById(id, scopeFilter), + embed: (text) => embedder.embedPassage(text), + completeJson: (prompt, label, system, temperature) => llmClient.completeJson(prompt, label, system, temperature), + log: (message) => console.warn(message), + confirmCost: async (message) => { + console.log(`\n${message}`); + return confirm("Proceed with these LLM calls? Type YES to continue: "); + }, + confirmApply: async (message, clusters) => { + console.log(`\n${formatConsolidatePlanForDisplay(clusters)}`); + console.log(`\n${message}`); + return confirm("Type YES to apply: "); + }, + onAudit: mdMirror + ? async (audit) => { + const summary = `${audit.action} survivor=${audit.survivorId.slice(0, 8)} absorbed=${audit.absorbedIds.map((id) => id.slice(0, 8)).join(",")} reason="${audit.reason}"`; + await mdMirror({ text: summary, category: "consolidation", scope: audit.scope, timestamp: Date.now() }, { source: `memory-consolidate:${audit.action}`, agentId: options.agent }); + } + : undefined, + }, { + scope, + category: options.category, + sinceMs, + includeReflectionSlices: options.includeReflectionSlices, + apply: options.apply === true, + autoConfirm: options.yes === true, + settledFingerprints: new Set(settledLedger[scope] ?? []), + }); + if (result.status === "aborted") { + const declined = (result.abortReason ?? "").includes("cost gate declined"); + if (declined) { + console.log(`consolidate: cancelled at the cost gate — no LLM calls were made.`); + console.log(`Pass --yes to skip this prompt (e.g. for automation).`); + } + else { + console.error(`consolidate: aborted -- ${result.abortReason}`); + if (result.costPreview) { + console.error(formatConsolidateCostPreview(result.costPreview)); + } + console.error(`Pass --yes to skip this prompt (e.g. for automation), or re-run interactively and type YES.`); + } + process.exit(1); + } + console.log(`Scanned ${pluralCount(result.scanned, "row")}, ${result.eligible} eligible for consolidation.`); + const settledNote = result.settledSkipped > 0 ? ` (${result.settledSkipped} settled in previous runs)` : ""; + if (result.clusters.length === 0) { + console.log(`0 candidates${settledNote} — nothing to consolidate.\n`); + } + else { + console.log(`Decided ${pluralCount(result.clusters.length, "cluster")}${settledNote}:\n`); + } + for (const cluster of result.clusters) { + if (cluster.malformed) { + const label = cluster.failure === "call-failed" ? "undecided: LLM call failed" : "skipped: malformed verdict"; + console.log(` [${label}] ${pluralCount(cluster.memberIds.length, "row")}`); + for (const text of cluster.memberTexts) + console.log(` - "${text}"`); + continue; + } + const blockedNote = cluster.blocked === "append-only-shield" ? " — BLOCKED by append-only shield (not applied)" : ""; + console.log(` [${cluster.verdict.verdict}] ${pluralCount(cluster.memberIds.length, "row")} — ${cluster.verdict.reason}${blockedNote}`); + for (const text of cluster.memberTexts) + console.log(` - "${text}"`); + } + if (result.staleSkipped.length > 0) { + console.log(`\n${pluralCount(result.staleSkipped.length, "cluster")} skipped: changed since the plan was built (stale).`); + } + if (result.status === "completed" && settledLedgerPath) { + await saveConsolidateSettledLedger(settledLedgerPath, settledLedger, scope, result.newlySettled); + } + if (!result.executed) { + if (!options.apply) { + console.log(`\nNo changes applied.`); + } + return; + } + const failureNotes = []; + if (result.skippedMalformed > 0) + failureNotes.push(`${pluralCount(result.skippedMalformed, "cluster")} skipped due to malformed verdicts`); + if (result.undecidedCallFailed > 0) + failureNotes.push(`${pluralCount(result.undecidedCallFailed, "cluster")} undecided because the decide call failed`); + console.log(`\nApplied ${pluralCount(result.applied.length, "action")}${failureNotes.length ? "; " + failureNotes.join("; ") : ""}.`); + } + catch (error) { + console.error("consolidate failed:", error); + process.exit(1); + } + }); } // ============================================================================ // Factory Function diff --git a/dist/index.js b/dist/index.js index baebf86a1..711b18078 100644 --- a/dist/index.js +++ b/dist/index.js @@ -2406,6 +2406,7 @@ const memoryLanceDBProPlugin = { onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), migrator, embedder, + mdMirror, llmClient: smartExtractor ? (() => { try { const llmAuth = config.llm?.auth || "api-key"; @@ -4066,10 +4067,15 @@ const memoryLanceDBProPlugin = { const now = new Date(params.timestampMs ?? Date.now()); const dateStr = now.toISOString().split("T")[0]; const timeStr = now.toISOString().split("T")[1].split(".")[0]; + // Session key/id stay out of `text`: it is the FTS index surface, and + // the `simple` tokenizer splits a key like + // `agent:main:cron::run:` on its punctuation — so every session + // summary ends up indexed under `agent`, `main`, `cron`, `run`. A query + // mentioning any of those then BM25-matches every session summary in the + // store regardless of content. Both ids are already recorded structurally + // in metadata below, so provenance is unaffected. const memoryText = [ `Session: ${dateStr} ${timeStr} UTC`, - `Session Key: ${params.sessionKey}`, - `Session ID: ${params.sessionId}`, `Source: ${params.source}`, "", "Conversation Summary:", @@ -4194,7 +4200,10 @@ const memoryLanceDBProPlugin = { return; } await mkdir(backupDir, { recursive: true }); - const allMemories = await store.list(undefined, undefined, 10000, 0); + // excludeInactive:false -- this is the automated backup dump and must + // keep full-dump semantics, including invalidated/superseded rows + // (item 6, PR #946). + const allMemories = await store.list(undefined, undefined, 10000, 0, { excludeInactive: false }); if (allMemories.length === 0) return; const dateStr = new Date().toISOString().split("T")[0]; diff --git a/dist/src/consolidate.js b/dist/src/consolidate.js new file mode 100644 index 000000000..2a24ae68b --- /dev/null +++ b/dist/src/consolidate.js @@ -0,0 +1,847 @@ +import { createHash } from "node:crypto"; +import { parseSmartMetadata, buildSmartMetadata, stringifySmartMetadata, appendRelation, deriveFactKey, isMemoryActiveAt, } from "./smart-metadata.js"; +import { APPEND_ONLY_CATEGORIES } from "./memory-categories.js"; +import { buildConsolidateBatchPrompt, buildConsolidateBatchMergePrompt, } from "./extraction-prompts.js"; +const REVERSAL_SIGNAL_PATTERN = /\b(no longer|not anymore|any ?more|stopped|quit|used to|former|discontinued|doesn'?t|don'?t|isn'?t|wasn'?t)\b/i; +const TOPIC_TOKEN_STOPWORDS = new Set([ + "user", "users", "prefer", "prefers", "preferred", "preference", "preferences", + "favorite", "favourite", "likes", "liked", "like", "dislikes", "dislike", + "drinking", "drinks", "drink", "drank", "still", "always", "anymore", "any", + "more", "longer", "stopped", "quit", "used", "no", "not", "the", "a", "an", + "of", "to", "and", "with", "their", "they", "was", "is", "are", "were", + "has", "have", "had", "will", "would", "their", "for", "at", "in", "on", + // Generic life-update narration: these appear across many unrelated life + // events/decisions and would otherwise let a single multi-topic narrative + // row bridge several unrelated topic clusters via incidental overlap. + "decided", "decide", "redesign", "redesigning", "relocate", + "relocating", "moved", "move", "moving", "changed", "change", "changing", + "switched", "switch", "started", "start", "starting", "continuing", + "continues", "testing", "tested", "experiment", "experimenting", "after", + "before", "now", "previously", "recently", "incident", "productivity", + "better", "correctly", "confirmed", "offered", "each", "record", "records", + "distinct", "fact", "facts", "note", "notes", "update", "updates", "updated", + "from", "into", "this", "that", "these", "those", "it", "its", "them", +]); +function looksLikeReversal(text) { + return REVERSAL_SIGNAL_PATTERN.test(text); +} +function extractTopicTokens(text) { + const words = text.toLowerCase().match(/[a-z0-9][a-z0-9'-]{2,}/g) || []; + return new Set(words.filter((w) => !TOPIC_TOKEN_STOPWORDS.has(w))); +} +// Reversal statements are typically short ("User will no longer drink +// cola"); a long multi-fact narrative recap can mention almost every topic +// in a scope at once and would otherwise bridge unrelated clusters through +// incidental keyword overlap. Only short, single-topic-looking statements +// participate in the topic-overlap fallback. +const REVERSAL_TOPIC_LINK_MAX_LENGTH = 120; +function isEligibleForTopicLink(abstract) { + return abstract.length <= REVERSAL_TOPIC_LINK_MAX_LENGTH; +} +// Tokens match on exact equality or containment (one is a substring of the +// other, e.g. "cola" inside "coca-cola"), since brand/product names are +// routinely abbreviated across lanes. The shorter token must still be long +// enough (>= 4 chars) to keep an accidental short-token containment match +// from firing. +function tokensMatch(a, b) { + if (a === b) + return true; + const shorter = a.length <= b.length ? a : b; + const longer = a.length <= b.length ? b : a; + return shorter.length >= 4 && longer.includes(shorter); +} +function shareSignificantTopicToken(a, b) { + const tokensA = extractTopicTokens(a); + const tokensB = extractTopicTokens(b); + for (const tokenA of tokensA) { + for (const tokenB of tokensB) { + if (tokensMatch(tokenA, tokenB)) + return true; + } + } + return false; +} +// Fraction of the SMALLER topic-token set that matches the other set +// (0 when either side has no topic tokens at all). Two short statements +// about the same narrow fact typically share MOST of their significant +// words even when phrased completely differently across write lanes +// (e.g. "Favorite drink: cola" vs "Cola is what gets ordered most +// evenings" both reduce to essentially {"cola"}); two short statements +// about DIFFERENT facts rarely do, which is what keeps this fallback from +// bridging unrelated rows the way a single-shared-token check would. +function topicTokenOverlapRatio(a, b) { + const tokensA = extractTopicTokens(a); + const tokensB = extractTopicTokens(b); + if (tokensA.size === 0 || tokensB.size === 0) + return 0; + let matches = 0; + for (const tokenA of tokensA) { + for (const tokenB of tokensB) { + if (tokensMatch(tokenA, tokenB)) { + matches += 1; + break; + } + } + } + return matches / Math.min(tokensA.size, tokensB.size); +} +const NEAR_DUPLICATE_TOKEN_OVERLAP_RATIO = 0.6; +function cosineSimilarity(a, b) { + if (a.length === 0 || b.length === 0 || a.length !== b.length) + return 0; + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA === 0 || normB === 0) + return 0; + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} +export function buildConsolidateCandidate(entry) { + const meta = parseSmartMetadata(entry.metadata, entry); + const abstract = meta.l0_abstract || entry.text; + const factKey = meta.fact_key || deriveFactKey(meta.memory_category, abstract); + return { + entry, + memoryCategory: meta.memory_category, + abstract, + overview: meta.l1_overview || "", + content: meta.l2_content || entry.text, + factKey, + source: meta.source, + validFrom: meta.valid_from, + }; +} +function isDirectlyLinked(a, b, similarityThreshold) { + const va = a.entry.vector; + const vb = b.entry.vector; + if (va.length > 0 && vb.length > 0 && cosineSimilarity(va, vb) >= similarityThreshold) { + return true; + } + if (a.factKey && a.factKey === b.factKey) { + return true; + } + // Reflection-mapped rows carry no stored fact_key, and a naturally phrased + // reversal rarely follows the "[Merge key]: text" convention that + // deriveFactKey needs to align across lanes, so its derived key is + // effectively unique. Gate a topic-word-overlap fallback to rows that look + // like a reversal, in the same category, and short enough to plausibly be + // about one topic, so it only widens linking for the exact case cosine + + // fact_key miss, not for arbitrary unrelated or multi-topic narrative rows. + if ((looksLikeReversal(a.abstract) || looksLikeReversal(b.abstract)) && + a.memoryCategory && + a.memoryCategory === b.memoryCategory && + isEligibleForTopicLink(a.abstract) && + isEligibleForTopicLink(b.abstract) && + shareSignificantTopicToken(a.abstract, b.abstract)) { + return true; + } + // Cross-lane near-duplicate fallback: two short, same-category rows that + // are NOT reversal-shaped can still be the same fact stated by different + // write lanes (manual/auto-capture/reflection*), whose differing + // tokenization keeps cosine and fact_key from matching. Unlike the + // reversal fallback above (which only needs ONE shared token, since a + // reversal is inherently pointed at a specific fact), this case requires + // a MAJORITY of the smaller row's topic tokens to overlap, so two short + // but topically different statements don't bridge on a single + // incidental shared word. + if (a.memoryCategory && + a.memoryCategory === b.memoryCategory && + isEligibleForTopicLink(a.abstract) && + isEligibleForTopicLink(b.abstract) && + topicTokenOverlapRatio(a.abstract, b.abstract) >= NEAR_DUPLICATE_TOKEN_OVERLAP_RATIO) { + return true; + } + return false; +} +/** + * Seed-based clustering: for each not-yet-assigned row (in order), it + * becomes the seed of a new cluster, and every OTHER unassigned row joins + * that cluster only if it is DIRECTLY linked to the seed itself (cosine, + * fact_key, or the topic-overlap fallback) -- never transitively through + * another cluster member. Plain union-find (transitive closure) chains + * unrelated rows together whenever a series of only-moderately-similar + * pairs bridges them (row A links to B, B links to C, so A and C end up in + * one cluster even though A and C are never themselves similar); seed-based + * grouping caps that at a single hop from the seed, which is what keeps a + * handful of distinct topics from collapsing into one grab-bag cluster. + */ +export function clusterConsolidateCandidates(candidates, similarityThreshold) { + const n = candidates.length; + const assigned = new Array(n).fill(false); + const clusters = []; + for (let seedIdx = 0; seedIdx < n; seedIdx++) { + if (assigned[seedIdx]) + continue; + assigned[seedIdx] = true; + const cluster = [seedIdx]; + for (let j = 0; j < n; j++) { + if (assigned[j]) + continue; + if (isDirectlyLinked(candidates[seedIdx], candidates[j], similarityThreshold)) { + assigned[j] = true; + cluster.push(j); + } + } + if (cluster.length >= 2) + clusters.push(cluster); + } + return clusters; +} +export function chunkCluster(indices, maxSize) { + const chunks = []; + for (let i = 0; i < indices.length; i += maxSize) { + chunks.push(indices.slice(i, i + maxSize)); + } + return chunks; +} +export function parseConsolidateVerdict(raw, memberCount) { + if (!raw || typeof raw !== "object") + return null; + const obj = raw; + const verdict = obj.verdict; + if (verdict !== "skip" && verdict !== "merge" && verdict !== "supersede" && verdict !== "contradict") { + return null; + } + const reason = typeof obj.reason === "string" ? obj.reason : ""; + if (verdict === "skip" || verdict === "contradict") { + return { verdict, reason }; + } + const survivorIndex = Number(obj.survivor_index); + if (!Number.isInteger(survivorIndex) || survivorIndex < 1 || survivorIndex > memberCount) + return null; + const absorbedRaw = obj.absorbed_indices; + if (!Array.isArray(absorbedRaw) || absorbedRaw.length === 0) + return null; + const absorbedIndices = absorbedRaw.map((v) => Number(v)); + if (absorbedIndices.some((i) => !Number.isInteger(i) || i < 1 || i > memberCount || i === survivorIndex)) { + return null; + } + return { verdict, reason, survivorIndex, absorbedIndices }; +} +// Parses the batched decider's `{ verdicts: [...] }` response into a +// clusterIndex -> verdict map. Fails closed PER CLUSTER: an entry with an +// unrecognized/duplicate cluster_index, or a malformed verdict shape for its +// own member count, is simply dropped rather than discarding the whole +// batch -- callers treat a missing clusterIndex as "skip this cluster" the +// same way a single malformed per-cluster response was already handled. +export function parseConsolidateBatchVerdicts(raw, units) { + const result = new Map(); + if (!raw || typeof raw !== "object") + return result; + const verdictsRaw = raw.verdicts; + if (!Array.isArray(verdictsRaw)) + return result; + const memberCountByCluster = new Map(units.map((u) => [u.clusterIndex, u.memberCount])); + for (const entry of verdictsRaw) { + if (!entry || typeof entry !== "object") + continue; + const clusterIndex = Number(entry.cluster_index); + if (!Number.isInteger(clusterIndex) || !memberCountByCluster.has(clusterIndex)) + continue; + if (result.has(clusterIndex)) + continue; + const verdict = parseConsolidateVerdict(entry, memberCountByCluster.get(clusterIndex)); + if (!verdict) + continue; + result.set(clusterIndex, verdict); + } + return result; +} +// ============================================================================ +// Item 7: LLM-cost gate. Clustering is free (local cosine + fact_key/topic +// linking); the only paid calls are the one batched decider call and one +// batched merge-content call (chunked past CONSOLIDATE_MERGE_BATCH_MAX_SIZE) +// covering every unit that MIGHT turn out to be a merge verdict, since item 8 +// moves merge-content generation into the plan phase. Both are knowable from +// clustering alone, before any LLM call is made -- which is what lets the +// gate sit ahead of the decide call and cover dry-runs as well as --apply. +// ============================================================================ +/** Max merge jobs written in one batched merge-content LLM call; larger batches are chunked. */ +export const CONSOLIDATE_MERGE_BATCH_MAX_SIZE = 10; +export function computeConsolidateCostPreview(units) { + const maxMergeJobs = units.length; + return { + clusterCount: units.length, + maxMergeJobs, + maxMergeContentCalls: Math.ceil(maxMergeJobs / CONSOLIDATE_MERGE_BATCH_MAX_SIZE), + }; +} +export function pluralCount(count, noun) { + return `${count} ${noun}${count === 1 ? "" : "s"}`; +} +export function formatConsolidateCostPreview(preview) { + const base = `${pluralCount(preview.clusterCount, "cluster")} -> 1 batched decider call`; + if (preview.maxMergeJobs === 0) + return base; + return `${base} + worst case ${pluralCount(preview.maxMergeContentCalls, "batched merge-content call")} covering ${pluralCount(preview.maxMergeJobs, "merge job")}`; +} +export function formatConsolidatePlanForDisplay(clusters) { + const actionable = clusters.filter((c) => c.action); + const blocked = clusters.filter((c) => c.blocked === "append-only-shield"); + const noAction = clusters.filter((c) => !c.action && !c.blocked && !c.malformed && c.verdict); + if (actionable.length === 0 && blocked.length === 0 && noAction.length === 0) { + return "No actionable clusters in this plan."; + } + const lines = [ + `Plan: ${pluralCount(actionable.length, "actionable cluster")}, ${blocked.length} blocked, ${pluralCount(noAction.length, "skip")}`, + ]; + for (const cluster of actionable) { + lines.push(` [${cluster.action}] cluster ${cluster.clusterIndex} — ${cluster.verdict.reason}`); + lines.push(` members: ${cluster.memberIds.join(", ")}`); + lines.push(` survivor: ${cluster.survivorId}`); + if (cluster.absorbedIds?.length) { + lines.push(` absorbed: ${cluster.absorbedIds.join(", ")}`); + } + if (cluster.action === "merge" && cluster.mergedContent) { + lines.push(` merged abstract: ${cluster.mergedContent.abstract}`); + lines.push(` merged overview: ${cluster.mergedContent.overview}`); + lines.push(` merged content: ${cluster.mergedContent.content}`); + } + } + for (const cluster of blocked) { + lines.push(` [${cluster.verdict.verdict} — BLOCKED by append-only shield, will NOT be applied] cluster ${cluster.clusterIndex} — ${cluster.verdict.reason}`); + lines.push(` members: ${cluster.memberIds.join(", ")}`); + for (const text of cluster.memberTexts) { + lines.push(` - "${text}"`); + } + } + for (const cluster of noAction) { + lines.push(` [${cluster.verdict.verdict}] cluster ${cluster.clusterIndex} — ${cluster.verdict.reason}`); + for (const text of cluster.memberTexts) { + lines.push(` - "${text}"`); + } + } + return lines.join("\n"); +} +/** + * Item 8: pure content generation for merge verdicts -- one batched + * `consolidate-merge-batch` completion per chunk of up to + * CONSOLIDATE_MERGE_BATCH_MAX_SIZE merge verdicts (each job folds ALL of a + * verdict's absorbed members into its survivor in one output), plus one + * re-embed per job, with NO store writes. Called at PLAN-BUILD time (dry-run + * or --apply alike), so execution later can be pure store operations that + * never regenerate content and never call the LLM again. + * + * Per-item fail-closed: a response entry that is missing or malformed + * degrades ONLY that job to the survivor's own unmodified content -- exactly + * what the sequential per-member fold produced when its completions came + * back null -- and a chunk whose call itself fails degrades every job in + * that chunk the same way. Never throws, never fans back out into per-member + * LLM calls. + */ +async function buildMergePlanContentsBatch(deps, jobs, log) { + const out = new Array(jobs.length); + for (let chunkStart = 0; chunkStart < jobs.length; chunkStart += CONSOLIDATE_MERGE_BATCH_MAX_SIZE) { + const chunk = jobs.slice(chunkStart, chunkStart + CONSOLIDATE_MERGE_BATCH_MAX_SIZE); + const prompt = buildConsolidateBatchMergePrompt(chunk.map(({ members, verdict }) => { + const survivor = members[verdict.survivorIndex - 1]; + return { + category: survivor.memoryCategory || "preferences", + existing: { + abstract: survivor.abstract, + overview: survivor.overview, + content: survivor.content, + }, + additions: verdict.absorbedIndices.map((idx) => { + const absorbed = members[idx - 1]; + return { + abstract: absorbed.abstract, + overview: absorbed.overview, + content: absorbed.content, + }; + }), + }; + })); + const byIndex = new Map(); + try { + const raw = await deps.completeJson(prompt.user, "consolidate-merge-batch", prompt.system); + for (const entry of raw && Array.isArray(raw.results) ? raw.results : []) { + if (!entry || typeof entry.index !== "number") + continue; + byIndex.set(entry.index, entry); + } + } + catch (err) { + log?.(`memory-consolidate: batched merge-content call failed, keeping survivor content for ${chunk.length} job(s): ${String(err)}`); + } + for (let i = 0; i < chunk.length; i++) { + const { members, verdict } = chunk[i]; + const survivor = members[verdict.survivorIndex - 1]; + const entry = byIndex.get(i + 1); + const usable = entry && + typeof entry.abstract === "string" && + entry.abstract.trim().length > 0 && + typeof entry.overview === "string" && + typeof entry.content === "string"; + if (!usable) { + log?.("memory-consolidate: missing or malformed merge-content entry, keeping survivor content for this job"); + } + const abstract = usable ? entry.abstract : survivor.abstract; + const overview = usable ? entry.overview : survivor.overview; + const content = usable ? entry.content : survivor.content; + const vector = await deps.embed(`${abstract} ${content}`); + out[chunkStart + i] = { abstract, overview, content, vector }; + } + } + return out; +} +/** + * Item 8: pure store write for an already-planned merge verdict. Applies + * EXACTLY the precomputed content from `buildMergePlanContentsBatch` -- no LLM + * call, no regeneration, "apply exactly what was presented." + */ +async function writeMergeVerdict(deps, members, verdict, mergedContent, scopeFilter, now) { + const survivor = members[verdict.survivorIndex - 1]; + const { abstract, overview, content, vector } = mergedContent; + const absorbedIds = []; + for (const idx of verdict.absorbedIndices) { + absorbedIds.push(members[idx - 1].entry.id); + } + const patchedMeta = buildSmartMetadata(survivor.entry, { + l0_abstract: abstract, + l1_overview: overview, + l2_content: content, + }); + const auditedMeta = { + ...patchedMeta, + consolidation_audit: { action: "merge", absorbedIds, reason: verdict.reason, at: now }, + }; + await deps.update(survivor.entry.id, { text: abstract, vector, metadata: stringifySmartMetadata(auditedMeta) }, scopeFilter); + // Non-destructive: absorbed rows are soft-invalidated with the same + // primitive applySupersedeVerdict uses (invalidated_at + superseded_by + + // relations), not hard-deleted. Each absorbed row also gets its own + // consolidation_audit pointing back at the survivor, so its history is + // independently inspectable without cross-referencing the survivor's + // audit. No LLM verdict path may call a hard delete; hard delete stays an + // operator-only CLI command. + for (const idx of verdict.absorbedIndices) { + const absorbed = members[idx - 1]; + const existingMeta = parseSmartMetadata(absorbed.entry.metadata, absorbed.entry); + const invalidatedMeta = buildSmartMetadata(absorbed.entry, { + invalidated_at: now, + superseded_by: survivor.entry.id, + relations: appendRelation(existingMeta.relations, { type: "superseded_by", targetId: survivor.entry.id }), + }); + const auditedAbsorbedMeta = { + ...invalidatedMeta, + consolidation_audit: { action: "merge", survivorId: survivor.entry.id, reason: verdict.reason, at: now }, + }; + await deps.update(absorbed.entry.id, { metadata: stringifySmartMetadata(auditedAbsorbedMeta) }, scopeFilter); + } + return { action: "merge", survivorId: survivor.entry.id, absorbedIds, reason: verdict.reason, scope: survivor.entry.scope }; +} +async function applySupersedeVerdict(deps, members, verdict, scopeFilter, now) { + const survivor = members[verdict.survivorIndex - 1]; + const factKey = survivor.factKey || members[verdict.absorbedIndices[0] - 1].factKey || ""; + const absorbedIds = []; + for (const idx of verdict.absorbedIndices) { + const absorbed = members[idx - 1]; + const existingMeta = parseSmartMetadata(absorbed.entry.metadata, absorbed.entry); + const invalidatedMetadata = buildSmartMetadata(absorbed.entry, { + fact_key: factKey || existingMeta.fact_key, + invalidated_at: now, + superseded_by: survivor.entry.id, + relations: appendRelation(existingMeta.relations, { type: "superseded_by", targetId: survivor.entry.id }), + }); + await deps.update(absorbed.entry.id, { metadata: stringifySmartMetadata(invalidatedMetadata) }, scopeFilter); + absorbedIds.push(absorbed.entry.id); + } + // An append-only (events/cases) survivor is left byte-untouched: the shield + // admits it only because nothing gets written to it, so even the fact_key + // patch and audit annotation are skipped (the audit still lands in the plan + // report and journal mirror). + const survivorIsAppendOnly = Boolean(survivor.memoryCategory && APPEND_ONLY_CATEGORIES.has(survivor.memoryCategory)); + if (!survivorIsAppendOnly) { + const survivorMeta = parseSmartMetadata(survivor.entry.metadata, survivor.entry); + const patchedSurvivorMeta = buildSmartMetadata(survivor.entry, { + fact_key: factKey || survivorMeta.fact_key, + }); + const auditedMeta = { + ...patchedSurvivorMeta, + consolidation_audit: { action: "supersede", absorbedIds, reason: verdict.reason, at: now }, + }; + await deps.update(survivor.entry.id, { metadata: stringifySmartMetadata(auditedMeta) }, scopeFilter); + } + return { action: "supersede", survivorId: survivor.entry.id, absorbedIds, reason: verdict.reason, scope: survivor.entry.scope }; +} +const DEFAULT_SIMILARITY_THRESHOLD = 0.86; +const DEFAULT_CLUSTER_CAP = 8; +const DEFAULT_SCAN_LIMIT = 100_000; +function abortedResult(reason, scanned, eligible, costPreview, apply) { + return { + status: "aborted", + abortReason: reason, + scanned, + eligible, + costPreview, + clusters: [], + applied: [], + executed: false, + staleSkipped: [], + skippedMalformed: 0, + undecidedCallFailed: 0, + settledSkipped: 0, + newlySettled: [], + apply, + }; +} +/** + * Stable identity for a cluster in the settled ledger: the sorted member + * ids with each member's exact metadata string. Any member change (edit, + * merge, invalidation) or any membership change produces a different + * fingerprint, so a settled entry can never suppress a cluster whose + * content moved on. + */ +export function computeClusterFingerprint(members) { + const parts = members + .map((m) => `${m.id}\n${m.metadata ?? ""}`) + .sort() + .join(""); + return createHash("sha256").update(parts).digest("hex"); +} +/** + * Item 8 staleness guard: re-fetches every member of a plan entry and + * compares its metadata string against the plan-build-time snapshot. + * Missing row (disappeared) or changed metadata (mutated by someone else) + * both count as stale. Skips the check entirely (treats as fresh) when + * deps.getById isn't provided -- an opt-in safety net, not a hard + * requirement, so callers that don't need it don't have to wire it up. + */ +async function isClusterFresh(deps, entry, scopeFilter) { + if (!deps.getById) + return true; + for (const snapshot of entry.staleness) { + const current = await deps.getById(snapshot.id, scopeFilter); + if (!current) + return false; + if (current.metadata !== snapshot.metadata) + return false; + } + return true; +} +async function executePlan(deps, clusters, membersByCluster, scopeFilter, now) { + const applied = []; + const staleSkipped = []; + for (const entry of clusters) { + if (!entry.action || !entry.verdict) + continue; + const members = membersByCluster.get(entry.clusterIndex); + if (!members) + continue; + const fresh = await isClusterFresh(deps, entry, scopeFilter); + if (!fresh) { + staleSkipped.push({ clusterIndex: entry.clusterIndex, memberIds: entry.memberIds }); + deps.log?.(`memory-consolidate: cluster ${entry.clusterIndex} changed since the plan was built (stale); skipping, never partially applied`); + continue; + } + try { + const audit = entry.action === "merge" + ? await writeMergeVerdict(deps, members, entry.verdict, entry.mergedContent, scopeFilter, now) + : await applySupersedeVerdict(deps, members, entry.verdict, scopeFilter, now); + applied.push(audit); + await deps.onAudit?.(audit); + } + catch (err) { + deps.log?.(`memory-consolidate: failed to apply ${entry.action} verdict: ${String(err)}`); + } + } + return { applied, staleSkipped }; +} +export async function runConsolidate(deps, options) { + const now = options.now ?? Date.now(); + const scopeFilter = options.scopeFilter ?? [options.scope]; + const rawEntries = await deps.fetchRows(scopeFilter, now, DEFAULT_SCAN_LIMIT); + const filtered = rawEntries.filter((entry) => { + if (entry.category === "reflection" && !options.includeReflectionSlices) + return false; + if (options.sinceMs !== undefined && entry.timestamp < options.sinceMs) + return false; + return true; + }); + const candidates = filtered + .map(buildConsolidateCandidate) + .filter((candidate) => { + const meta = parseSmartMetadata(candidate.entry.metadata, candidate.entry); + if (!isMemoryActiveAt(meta, now)) + return false; + if (options.category && candidate.memoryCategory !== options.category) + return false; + return true; + }) + // Sort by row id (a stable key) before clustering, not just before + // building the prompt: clusterConsolidateCandidates' seed-based scan + // always picks the lowest surviving array index as the next seed, so a + // pre-sorted candidate array makes both which rows end up in the same + // cluster AND their order within it a pure function of the candidate + // SET -- independent of whatever order fetchRows happened to return + // this call, which store/DB internals don't guarantee is stable. + .sort((a, b) => (a.entry.id < b.entry.id ? -1 : a.entry.id > b.entry.id ? 1 : 0)); + const similarityThreshold = options.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD; + const clusterCap = options.clusterCap ?? DEFAULT_CLUSTER_CAP; + const clusterIndexGroups = clusterConsolidateCandidates(candidates, similarityThreshold); + const byId = (a, b) => a.entry.id < b.entry.id ? -1 : a.entry.id > b.entry.id ? 1 : 0; + // Flatten every cluster (and any cluster chunked past clusterCap) into a + // single ordered list of decision units first, so the decider can be + // asked about all of them in ONE completeJson call instead of one call + // per cluster. Members within a unit, and units themselves, are + // explicitly re-sorted by row id here too (belt-and-suspenders on top of + // the pre-clustering sort above) so prompt assembly never depends on + // clusterConsolidateCandidates' internal grouping order. + const units = []; + for (const group of clusterIndexGroups) { + const sortedGroup = [...group].sort((a, b) => byId(candidates[a], candidates[b])); + const chunks = chunkCluster(sortedGroup, clusterCap); + for (const chunkIndices of chunks) { + if (chunkIndices.length < 2) + continue; + units.push({ clusterIndex: units.length + 1, members: chunkIndices.map((i) => candidates[i]) }); + } + } + units.sort((a, b) => byId(a.members[0], b.members[0])); + // Convergence: clusters settled by a previous run (same members, same + // content) are dropped before the cost gate and the decider ever see + // them, so repeated runs over an unchanged store reach zero clusters. + const fingerprintByUnit = new Map(); + for (const unit of units) { + fingerprintByUnit.set(unit, computeClusterFingerprint(unit.members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata })))); + } + let settledSkipped = 0; + const activeUnits = units.filter((unit) => { + if (options.settledFingerprints?.has(fingerprintByUnit.get(unit))) { + settledSkipped += 1; + return false; + } + return true; + }); + units.length = 0; + units.push(...activeUnits); + units.forEach((unit, i) => { + unit.clusterIndex = i + 1; + }); + // Item 7: the cost gate sits here -- clustering above is free (local + // cosine + fact_key/topic linking), and everything below this point is + // the first LLM call onward. Skipped entirely when there's nothing to + // decide (nothing to confirm), and bypassed without ever calling + // confirmCost when autoConfirm (--yes) is set. A declined OR missing + // confirmCost is treated identically: a safe abort, never assumed consent. + let costPreview; + if (units.length > 0) { + costPreview = computeConsolidateCostPreview(units); + if (!options.autoConfirm) { + const message = formatConsolidateCostPreview(costPreview); + const proceed = deps.confirmCost ? await deps.confirmCost(message) : false; + if (!proceed) { + return abortedResult("cost gate declined (or no confirmCost dep and --yes not set): no LLM call was made", rawEntries.length, candidates.length, costPreview, options.apply); + } + } + } + const clusters = []; + const membersByCluster = new Map(); + const pendingMergeContent = []; + let skippedMalformed = 0; + let undecidedCallFailed = 0; + let decideCallFailed = false; + const newlySettled = []; + if (units.length > 0) { + const batchClusters = units.map((unit) => ({ + clusterIndex: unit.clusterIndex, + members: unit.members.map((m, i) => ({ + index: i + 1, + category: m.memoryCategory || "preferences", + abstract: m.abstract, + overview: m.overview, + content: m.content, + source: m.source, + timestamp: m.entry.timestamp, + validFrom: m.validFrom, + })), + })); + const prompt = buildConsolidateBatchPrompt(batchClusters); + const raw = await deps.completeJson(prompt.user, "consolidate-decide", prompt.system, 0); + decideCallFailed = raw === null || raw === undefined; + if (decideCallFailed) { + deps.log?.(`memory-consolidate: consolidate-decide call returned no response (provider error or timeout); ${units.length} cluster(s) left undecided`); + } + const verdictMap = !decideCallFailed + ? parseConsolidateBatchVerdicts(raw, units.map((u) => ({ clusterIndex: u.clusterIndex, memberCount: u.members.length }))) + : new Map(); + // Build the COMPLETE plan now, regardless of apply/dry-run -- + // every merge verdict gets its content generated here (moved from + // apply time), so execution later is pure store writes with zero + // further LLM calls. + for (const unit of units) { + const members = unit.members; + const verdict = verdictMap.get(unit.clusterIndex) ?? null; + const fingerprint = fingerprintByUnit.get(unit); + membersByCluster.set(unit.clusterIndex, members); + if (!verdict) { + if (decideCallFailed) { + undecidedCallFailed += 1; + } + else { + skippedMalformed += 1; + deps.log?.(`memory-consolidate: missing or malformed verdict for a cluster of ${members.length} rows, skipping`); + } + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict: null, + malformed: true, + failure: decideCallFailed ? "call-failed" : "malformed-verdict", + fingerprint, + action: null, + staleness: members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata })), + }); + continue; + } + const staleness = members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata })); + if (verdict.verdict === "skip" || verdict.verdict === "contradict") { + newlySettled.push(fingerprint); + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + fingerprint, + action: null, + staleness, + }); + continue; + } + const survivorCategory = members[verdict.survivorIndex - 1].memoryCategory; + const absorbedCategories = verdict.absorbedIndices.map((idx) => members[idx - 1].memoryCategory); + const survivorIsAppendOnly = Boolean(survivorCategory && APPEND_ONLY_CATEGORIES.has(survivorCategory)); + const absorbedTouchesAppendOnly = absorbedCategories.some((category) => category && APPEND_ONLY_CATEGORIES.has(category)); + // Append-only means invalidation-protection, not merge-immunity, and + // the protection is directional: absorbed rows are what get invalidated + // (and merge additionally rewrites the survivor's content), so an + // append-only row may never be absorbed, and may only be a merge + // survivor when every acted-upon row shares the identical append-only + // category (a genuine same-category duplicate). A supersede survivor is + // never written at all when it is append-only (applySupersedeVerdict + // skips even the audit annotation), so an append-only row superseding + // stale mutable rows leaves the append-only guarantee intact. + const isSameCategoryAppendOnlyMerge = verdict.verdict === "merge" && + survivorIsAppendOnly && + absorbedCategories.every((category) => category === survivorCategory); + const blockedByShield = verdict.verdict === "merge" + ? (survivorIsAppendOnly || absorbedTouchesAppendOnly) && !isSameCategoryAppendOnlyMerge + : absorbedTouchesAppendOnly; + if (blockedByShield) { + deps.log?.(`memory-consolidate: refusing to ${verdict.verdict} an append-only row (events/cases) outside a same-category duplicate merge; skipping this verdict`); + // A shield-blocked verdict is as settled as a skip: re-running the + // decider over the same unchanged members can only produce another + // blocked verdict. + newlySettled.push(fingerprint); + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + blocked: "append-only-shield", + fingerprint, + action: null, + staleness, + }); + continue; + } + const survivor = members[verdict.survivorIndex - 1]; + const absorbedIds = verdict.absorbedIndices.map((idx) => members[idx - 1].entry.id); + const cluster = { + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + fingerprint, + action: verdict.verdict === "merge" ? "merge" : "supersede", + survivorId: survivor.entry.id, + absorbedIds, + staleness, + }; + clusters.push(cluster); + if (verdict.verdict === "merge") { + pendingMergeContent.push({ cluster, members, verdict }); + } + } + // One batched merge-content call (chunk-capped) covers every merge + // verdict's plan content, moved out of the per-unit loop so the plan + // build spends ceil(M/CONSOLIDATE_MERGE_BATCH_MAX_SIZE) LLM calls + // instead of one call per absorbed member. + if (pendingMergeContent.length > 0) { + const contents = await buildMergePlanContentsBatch(deps, pendingMergeContent, deps.log); + pendingMergeContent.forEach((pending, i) => { + pending.cluster.mergedContent = contents[i]; + }); + } + } + const actionable = clusters.filter((c) => c.action); + // Item 8: direct --apply executes the plan immediately, no second prompt. + if (options.apply) { + const { applied, staleSkipped } = await executePlan(deps, actionable, membersByCluster, scopeFilter, now); + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied, + executed: true, + staleSkipped, + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: true, + }; + } + // Dry-run / interactive path: present the full plan, ask once, execute + // only on an explicit affirmative. A declined or missing confirmApply is + // a safe no-op -- the plan was built (and its LLM calls already spent), + // but nothing is written. + if (actionable.length > 0) { + const message = `${pluralCount(actionable.length, "cluster")} ready to apply. Apply these now? (YES/no)`; + const proceed = deps.confirmApply ? await deps.confirmApply(message, clusters) : false; + if (proceed) { + const { applied, staleSkipped } = await executePlan(deps, actionable, membersByCluster, scopeFilter, now); + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied, + executed: true, + staleSkipped, + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: false, + }; + } + } + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied: [], + executed: false, + staleSkipped: [], + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: false, + }; +} diff --git a/dist/src/extraction-prompts.js b/dist/src/extraction-prompts.js index b74c9b3d9..0b9e9315e 100644 --- a/dist/src/extraction-prompts.js +++ b/dist/src/extraction-prompts.js @@ -201,3 +201,183 @@ Return JSON: "content": "Merged full content" } `; } +export const CONSOLIDATE_MERGE_SYSTEM_PROMPT = `You are a memory consolidation merge writer. Merge two versions of the same memory into a single coherent record with all three levels (abstract, overview, content). + +Requirements: +- Remove duplicate information +- Keep the most up-to-date details +- Maintain a coherent narrative +- Keep code identifiers, URIs, and model names unchanged when they are proper nouns + +Return JSON only: +{ + "abstract": "Merged one-line abstract", + "overview": "Merged structured Markdown overview", + "content": "Merged full content" +}`; +// mapped/manual/legacy rows without a real overview/content commonly fall +// back to the raw abstract text in all three tiers (see +// src/smart-metadata.ts's parseSmartMetadata: l2_content falls back to raw +// text, l1_overview falls back to `- ${abstract}`). Printing that fact three +// times per member wastes cluster-listing space for no signal. +function hasThinTiers(m) { + const overviewIsDefault = m.overview === "" || m.overview === `- ${m.abstract}` || m.overview === m.abstract; + const contentIsDefault = m.content === m.abstract; + return overviewIsDefault && contentIsDefault; +} +function formatMemberHeader(m) { + const parts = [`${m.index}. [${m.category}]`]; + if (m.source) + parts.push(` (source: ${m.source})`); + if (m.timestamp !== undefined) { + parts.push(`, timestamp: ${new Date(m.timestamp).toISOString()}`); + if (m.validFrom !== undefined && m.validFrom !== m.timestamp) { + parts.push(`, valid_from: ${new Date(m.validFrom).toISOString()}`); + } + } + return parts.join(""); +} +function formatMemberTiers(m) { + if (hasThinTiers(m)) { + return `Fact: ${m.abstract}`; + } + return `Abstract: ${m.abstract}\nOverview: ${m.overview}\nContent: ${m.content}`; +} +export function buildConsolidatePrompt(members) { + const system = `You are a memory consolidation decider. You are given a cluster of existing memories that were flagged as likely related, either by embedding similarity or by sharing a topic key. Decide how to reconcile the ACTIONABLE rows in this cluster. You do NOT have to act on every row: survivor_index and absorbed_indices only need to cover the rows you are deciding about. Any row you leave out of both is simply left untouched — this is expected and correct whenever a cluster mixes actionable duplicates or reversals with unrelated or append-only rows. + +Return exactly one verdict, scoped to whichever rows it actually applies to: +- skip: none of the rows in this cluster need any action. Use this only when nothing here is a duplicate, reversal, or contradiction. +- merge: two or more rows are duplicates or near-duplicates of the same fact. Pick the row with the best-quality, most complete text as the survivor and list only the true duplicates as absorbed. +- supersede: one row is a newer fact or an explicit reversal that replaces one or more older rows describing the same fact (for example, a decision to stop doing something an older row describes). The survivor is the newer/reversal row; list only the rows it actually replaces as absorbed. Supersede is NOT destructive: absorbed rows are never deleted. They are kept as an auditable historical record and simply marked as no longer current, exactly like SUPERSEDE in ordinary dedup decisions ("the same mutable fact has changed over time; keep the old memory as historical but no longer current"). Use supersede whenever a row states that a fact from an older row has changed, even if that only applies to part of the cluster. +- contradict: two or more rows conflict and it is not clear which one is correct. Flag this for human review. No destructive action. + +"events" and "cases" categories are append-only: they can never be superseded or contradicted (append-only means invalidation-protection, not merge-immunity). A merge must never mix an append-only row with a non-append-only row, or with a different append-only category. The one exception: near-identical duplicate rows within the SAME append-only category (for example two "events" rows describing the exact same occurrence, or two "cases" rows describing the exact same problem/solution) may still be merged like any other true duplicate. Outside that same-category duplicate case, leave append-only rows out of absorbed_indices, with one directional exception: an append-only row MAY serve as the supersede survivor_index when every absorbed row is non-append-only — the append-only row itself is never written, only the stale mutable rows get marked no longer current. None of this ever blocks you from merging or superseding the OTHER, actionable rows in the same cluster. + +Rows in DIFFERENT non-append-only categories (profile, preferences, entities, patterns) are fully actionable against each other — differing categories alone are never a reason to skip. Merge them when they state the same fact, choosing the more authoritative category's row as survivor (for identity facts like the user's name, profile over preferences); supersede when they conflict about the same fact, choosing the factually current row as survivor. Factual currency always decides supersede direction: never make a stale row the survivor for category reasons, and when the stale side is append-only (so it cannot be absorbed), use skip rather than a wrong-direction supersede. + +Source legend: legacy = pre-smart-format rows, manual = operator memory_store saves, auto-capture = extraction lane, reflection* = mirror lanes; manual rows are operator-authored and strong survivor candidates. + +Each member below also shows its timestamp (and valid_from when it differs) — use these to judge supersede recency explicitly rather than inferring it from wording alone. + +Return JSON only: +{ + "verdict": "skip|merge|supersede|contradict", + "survivor_index": 1, + "absorbed_indices": [2, 3], + "reason": "short explanation" +} + +Only include survivor_index and absorbed_indices for merge or supersede. survivor_index and every entry in absorbed_indices must be one of the row numbers shown below. absorbed_indices must never contain an append-only (events/cases) row — unless the verdict is merge and every row in survivor_index/absorbed_indices shares the exact same append-only category. An append-only row may appear as survivor_index only for that same-category duplicate merge, or for a supersede whose absorbed rows are all non-append-only.`; + const user = `Cluster members:\n\n${members + .map((m) => `${formatMemberHeader(m)}\n${formatMemberTiers(m)}`) + .join("\n\n")}`; + return { system, user }; +} +// Same decider semantics as buildConsolidatePrompt, but scoped to decide +// N independent clusters in a single call: one LLM round-trip per +// consolidate run instead of one per cluster. Each cluster is decided +// independently -- a verdict for one cluster must never be influenced by +// another cluster's rows -- and the response is a JSON array with one +// verdict object per cluster, tagged by cluster_index so a malformed entry +// for one cluster can be dropped without discarding the others' verdicts. +export function buildConsolidateBatchPrompt(clusters) { + const system = `You are a memory consolidation decider. You are given multiple independent clusters of existing memories, each flagged as likely related within itself, either by embedding similarity or by sharing a topic key. Decide how to reconcile the ACTIONABLE rows in EACH cluster independently -- a decision about one cluster must never be influenced by another cluster's rows. You do NOT have to act on every row in a cluster: survivor_index and absorbed_indices only need to cover the rows you are deciding about within that cluster. Any row you leave out of both is simply left untouched -- this is expected and correct whenever a cluster mixes actionable duplicates or reversals with unrelated or append-only rows. + +Return exactly one verdict per cluster, scoped to whichever rows it actually applies to: +- skip: none of the rows in this cluster need any action. Use this only when nothing here is a duplicate, reversal, or contradiction. +- merge: two or more rows are duplicates or near-duplicates of the same fact. Pick the row with the best-quality, most complete text as the survivor and list only the true duplicates as absorbed. +- supersede: one row is a newer fact or an explicit reversal that replaces one or more older rows describing the same fact (for example, a decision to stop doing something an older row describes). The survivor is the newer/reversal row; list only the rows it actually replaces as absorbed. Supersede is NOT destructive: absorbed rows are never deleted. They are kept as an auditable historical record and simply marked as no longer current, exactly like SUPERSEDE in ordinary dedup decisions ("the same mutable fact has changed over time; keep the old memory as historical but no longer current"). Use supersede whenever a row states that a fact from an older row has changed, even if that only applies to part of the cluster. +- contradict: two or more rows conflict and it is not clear which one is correct. Flag this for human review. No destructive action. + +Decision criteria: apply these checks in order for the rows in each cluster. +1. Do two or more rows say the same thing, with no row stating a newer fact, a change, or a reversal? -> merge. +2. Does one row explicitly state a fact has changed, ended, or reversed relative to another row (wording like "no longer", "stopped", "switched to", or simply a materially later timestamp describing a different state of the same fact)? -> supersede. +3. Do two or more rows assert mutually exclusive facts with no textual or temporal signal indicating which one is current? -> contradict. +4. None of the above apply to any rows in this cluster? -> skip. +When it is genuinely ambiguous whether a pair of rows should be merged or superseded, prefer supersede: it is the safer, fully-reversible choice, since a superseded row is retained as historical record rather than combined away into a single new record. + +"events" and "cases" categories are append-only: they can never be superseded or contradicted (append-only means invalidation-protection, not merge-immunity). A merge must never mix an append-only row with a non-append-only row, or with a different append-only category. The one exception: near-identical duplicate rows within the SAME append-only category (for example two "events" rows describing the exact same occurrence, or two "cases" rows describing the exact same problem/solution) may still be merged like any other true duplicate. Outside that same-category duplicate case, leave append-only rows out of absorbed_indices, with one directional exception: an append-only row MAY serve as the supersede survivor_index when every absorbed row is non-append-only -- the append-only row itself is never written, only the stale mutable rows get marked no longer current. None of this ever blocks you from merging or superseding the OTHER, actionable rows in the same cluster. + +Rows in DIFFERENT non-append-only categories (profile, preferences, entities, patterns) are fully actionable against each other -- differing categories alone are never a reason to skip. Merge them when they state the same fact, choosing the more authoritative category's row as survivor (for identity facts like the user's name, profile over preferences); supersede when they conflict about the same fact, choosing the factually current row as survivor. Factual currency always decides supersede direction: never make a stale row the survivor for category reasons, and when the stale side is append-only (so it cannot be absorbed), use skip rather than a wrong-direction supersede. + +Source legend: legacy = pre-smart-format rows, manual = operator memory_store saves, auto-capture = extraction lane, reflection* = mirror lanes; manual rows are operator-authored and strong survivor candidates. + +Each member below also shows its timestamp (and valid_from when it differs) -- use these to judge supersede recency explicitly rather than inferring it from wording alone. + +Return JSON only: +{ + "verdicts": [ + { "cluster_index": 1, "verdict": "skip|merge|supersede|contradict", "survivor_index": 1, "absorbed_indices": [2, 3], "reason": "short explanation" } + ] +} + +Include exactly one verdict object per cluster listed below, each tagged with the matching cluster_index. Only include survivor_index and absorbed_indices for merge or supersede. survivor_index and every entry in absorbed_indices are row numbers scoped to that cluster's own member list. absorbed_indices must never contain an append-only (events/cases) row -- unless the verdict is merge and every row in survivor_index/absorbed_indices shares the exact same append-only category. An append-only row may appear as survivor_index only for that same-category duplicate merge, or for a supersede whose absorbed rows are all non-append-only.`; + const user = clusters + .map((c) => `Cluster ${c.clusterIndex} members:\n\n${c.members + .map((m) => `${formatMemberHeader(m)}\n${formatMemberTiers(m)}`) + .join("\n\n")}`) + .join("\n\n===\n\n"); + return { system, user }; +} +/** + * Formats one labelled field for a numbered prompt block: the field on its + * own 3-space-indented line, multi-line values split per line with any + * leading markdown list-marker run (`- ` / `* `, repeated) stripped while + * the line's own inner indentation is kept, and every continuation line + * indented under the block. Other content markdown (e.g. `##` headings) is + * deliberately left as-is. + */ +function formatIndentedFieldLines(label, value) { + const valueLines = String(value ?? "") + .split("\n") + .map((line) => line.replace(/^(\s*)(?:[-*] )+/, "$1")); + const lines = [` ${label}: ${valueLines[0]}`]; + for (const continuation of valueLines.slice(1)) { + lines.push(` ${continuation}`); + } + return lines; +} +/** + * Batched variant of the consolidate merge writer prompt: one LLM call + * writes every numbered merge job. Each job carries its survivor ("Existing + * memory") and every absorbed member folding into it ("New information"); + * merge requirements match CONSOLIDATE_MERGE_SYSTEM_PROMPT verbatim — only + * the call topology changes from one call per absorbed member to one call + * per batch of merge verdicts. + */ +export function buildConsolidateBatchMergePrompt(jobs) { + const system = `You are a memory consolidation merge writer. Merge each numbered job below into a single coherent record with all three levels (abstract, overview, content). For each job, merge every "New information" section into that job's "Existing memory"; never mix content across jobs. + +Requirements: +- Remove duplicate information +- Keep the most up-to-date details +- Maintain a coherent narrative +- Keep code identifiers, URIs, and model names unchanged when they are proper nouns + +Return JSON only, with exactly one entry per job, in this shape: +{ + "results": [ + { "index": 1, "abstract": "Merged one-line abstract", "overview": "Merged structured Markdown overview", "content": "Merged full content" } + ] +} + +- "index" is the job's number in the batch below.`; + const blocks = jobs.map((job, i) => { + const lines = [`${i + 1}. Category: ${job.category}`, ` Existing memory:`]; + lines.push(...formatIndentedFieldLines("Abstract", job.existing.abstract)); + lines.push(...formatIndentedFieldLines("Overview", job.existing.overview)); + lines.push(...formatIndentedFieldLines("Content", job.existing.content)); + job.additions.forEach((addition, j) => { + lines.push(job.additions.length > 1 ? ` New information ${j + 1}:` : ` New information:`); + lines.push(...formatIndentedFieldLines("Abstract", addition.abstract)); + lines.push(...formatIndentedFieldLines("Overview", addition.overview)); + lines.push(...formatIndentedFieldLines("Content", addition.content)); + }); + return lines.join("\n"); + }); + const user = `Merge jobs: + +${blocks.join("\n\n")}`; + return { system, user }; +} diff --git a/dist/src/llm-client.js b/dist/src/llm-client.js index a44f0b780..9437e236f 100644 --- a/dist/src/llm-client.js +++ b/dist/src/llm-client.js @@ -4,6 +4,7 @@ */ import OpenAI from "openai"; import { buildOauthEndpoint, extractOutputTextFromSse, loadOAuthSession, needsRefresh, normalizeOauthModel, refreshOAuthSession, saveOAuthSession, } from "./llm-oauth.js"; +const DEFAULT_SYSTEM_PROMPT = "You are a memory extraction assistant. Always respond with valid JSON only."; /** * Extract JSON from an LLM response that may be wrapped in markdown fences * or contain surrounding text. @@ -185,7 +186,7 @@ function createApiKeyClient(config, log, warnLog) { }); let lastError = null; return { - async completeJson(prompt, label = "generic") { + async completeJson(prompt, label = "generic", systemPrompt, temperature) { lastError = null; try { const request = { @@ -193,11 +194,11 @@ function createApiKeyClient(config, log, warnLog) { messages: [ { role: "system", - content: "You are a memory extraction assistant. Always respond with valid JSON only.", + content: systemPrompt ?? DEFAULT_SYSTEM_PROMPT, }, { role: "user", content: prompt }, ], - temperature: 0.1, + temperature: temperature ?? 0.1, ...(shouldDisableReasoningForJson(config.model) ? { chat_template_kwargs: { enable_thinking: false } } : {}), @@ -294,7 +295,7 @@ function createOauthClient(config, log, warnLog) { return session; } return { - async completeJson(prompt, label = "generic") { + async completeJson(prompt, label = "generic", systemPrompt, _temperature) { lastError = null; try { const session = await getSession(); @@ -314,7 +315,7 @@ function createOauthClient(config, log, warnLog) { signal, body: JSON.stringify({ model: normalizeOauthModel(config.model), - instructions: "You are a memory extraction assistant. Always respond with valid JSON only.", + instructions: systemPrompt ?? DEFAULT_SYSTEM_PROMPT, input: [ { role: "user", diff --git a/dist/src/store.js b/dist/src/store.js index dde800e87..d912efbfd 100644 --- a/dist/src/store.js +++ b/dist/src/store.js @@ -1451,7 +1451,9 @@ export class MemoryStore { const safeLimit = clampInt(limit, 1, 20); // Over-fetch more aggressively when filtering inactive records, // because superseded historical rows can crowd out active ones. - const inactiveFilter = options?.excludeInactive ?? false; + // excludeInactive defaults to true: invalidated/superseded rows are + // invisible unless a caller opts out explicitly (item 6, PR #946). + const inactiveFilter = options?.excludeInactive ?? true; const overFetchMultiplier = inactiveFilter ? 20 : 10; const fetchLimit = Math.min(safeLimit * overFetchMultiplier, 200); if (this.disableNativeCosine && !this.nativeCosineFallbackLogged) { @@ -1524,7 +1526,8 @@ export class MemoryStore { if (isExplicitDenyAllScopeFilter(scopeFilter)) return []; const safeLimit = clampInt(limit, 1, 20); - const inactiveFilter = options?.excludeInactive ?? false; + // excludeInactive defaults to true: see vectorSearch above (item 6, PR #946). + const inactiveFilter = options?.excludeInactive ?? true; // Over-fetch when filtering inactive records to avoid crowding const fetchLimit = inactiveFilter ? Math.min(safeLimit * 20, 200) : safeLimit; if (!this.ftsIndexCreated && !(await this.refreshFtsSupportFromTable())) { @@ -1622,8 +1625,9 @@ export class MemoryStore { metadata: row.metadata || "{}", }; const metadata = parseSmartMetadata(entry.metadata, entry); - // Skip inactive (superseded) records when requested - if (options?.excludeInactive && !isMemoryActiveAt(metadata)) { + // Skip inactive (superseded) records unless explicitly opted out + // (excludeInactive defaults to true -- item 6, PR #946). + if ((options?.excludeInactive ?? true) && !isMemoryActiveAt(metadata)) { continue; } const score = scoreLexicalHit(trimmedQuery, [ @@ -1692,7 +1696,7 @@ export class MemoryStore { return true; }); } - async list(scopeFilter, category, limit = 20, offset = 0) { + async list(scopeFilter, category, limit = 20, offset = 0, options) { await this.ensureInitialized(); if (isExplicitDenyAllScopeFilter(scopeFilter)) return []; @@ -1732,9 +1736,15 @@ export class MemoryStore { timestamp: normalizeMemoryTimestamp(row.timestamp, 0), metadata: row.metadata || "{}", })); + // excludeInactive defaults to true: invalidated/superseded rows are + // invisible to list() unless a caller opts out explicitly (item 6, PR #946). + const excludeInactive = options?.excludeInactive ?? true; + const activeEntries = excludeInactive + ? entries.filter((entry) => isMemoryActiveAt(parseSmartMetadata(entry.metadata, entry))) + : entries; return (category - ? entries.filter((entry) => matchesMemoryCategoryFilter(entry.category, category, entry.metadata)) - : entries) + ? activeEntries.filter((entry) => matchesMemoryCategoryFilter(entry.category, category, entry.metadata)) + : activeEntries) .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)) .slice(offset, offset + limit); } @@ -1756,6 +1766,7 @@ export class MemoryStore { if (isExplicitDenyAllScopeFilter(scopeFilter)) { return { totalCount: 0, + liveCount: 0, scopeCounts: {}, categoryCounts: {}, }; @@ -1768,17 +1779,28 @@ export class MemoryStore { conditions.push(`((${scopeConditions}) OR scope IS NULL)`); } const applyConditions = (query) => conditions.length > 0 ? query.where(conditions.join(" AND ")) : query; - const results = await this.queryRowsWithProjectionFallback(applyConditions, ["scope", "category"]); + // scopeCounts/categoryCounts stay blended (total, historical record + // included) -- only the top-level total/live split is added here, per + // item 6 (PR #946): "report a live vs total split rather than one + // blended count." + const results = await this.queryRowsWithProjectionFallback(applyConditions, ["scope", "category", "metadata", "timestamp"]); const scopeCounts = {}; const categoryCounts = {}; + let liveCount = 0; for (const row of results) { const scope = row.scope ?? "global"; const category = row.category; scopeCounts[scope] = (scopeCounts[scope] || 0) + 1; categoryCounts[category] = (categoryCounts[category] || 0) + 1; + const metadata = parseSmartMetadata(row.metadata || "{}", { + timestamp: normalizeMemoryTimestamp(row.timestamp, 0), + }); + if (isMemoryActiveAt(metadata)) + liveCount += 1; } return { totalCount: results.length, + liveCount, scopeCounts, categoryCounts, }; @@ -2189,7 +2211,7 @@ export class MemoryStore { * omitted from `list()` for performance, but compaction needs them for * cosine-similarity clustering. */ - async fetchForCompaction(maxTimestamp, scopeFilter, limit = 200) { + async fetchForCompaction(maxTimestamp, scopeFilter, limit = 200, options) { await this.ensureInitialized(); const conditions = [timestampBeforePredicate("timestamp", maxTimestamp)]; if (scopeFilter && scopeFilter.length > 0) { @@ -2203,7 +2225,7 @@ export class MemoryStore { .query() .where(whereClause) .toArray(); - return results + const entries = results .map((row) => ({ id: row.id, text: row.text, @@ -2213,7 +2235,14 @@ export class MemoryStore { importance: clampImportance(Number(row.importance)), timestamp: normalizeMemoryTimestamp(row.timestamp, 0), metadata: row.metadata || "{}", - })) + })); + // excludeInactive defaults to true: a background compactor or + // consolidate run must not cluster already-dead rows (item 6, PR #946). + const excludeInactive = options?.excludeInactive ?? true; + const activeEntries = excludeInactive + ? entries.filter((entry) => isMemoryActiveAt(parseSmartMetadata(entry.metadata, entry))) + : entries; + return activeEntries .sort((a, b) => b.timestamp - a.timestamp) .slice(0, limit); } diff --git a/dist/src/tools.js b/dist/src/tools.js index 02b83e060..964f7fb4e 100644 --- a/dist/src/tools.js +++ b/dist/src/tools.js @@ -1734,9 +1734,10 @@ export function registerMemoryListTool(api, context) { offset: Type.Optional(Type.Number({ description: "Number of memories to skip (default: 0)", })), + includeInvalidated: Type.Optional(Type.Boolean({ description: "Include invalidated/superseded rows (default false)." })), }), async execute(_toolCallId, params, _signal, _onUpdate, runtimeCtx) { - const { limit = 10, scope, category, offset = 0, } = params; + const { limit = 10, scope, category, offset = 0, includeInvalidated = false, } = params; try { const safeLimit = clampInt(limit, 1, 50); const safeOffset = clampInt(offset, 0, 1000); @@ -1744,7 +1745,7 @@ export function registerMemoryListTool(api, context) { const resolvedScopes = resolveReadableToolScopeFilter(context.scopeManager, agentId, scope); const { scopeFilter } = resolvedScopes; const ignoredScopeNotice = formatIgnoredScopeNotice(resolvedScopes); - const entries = await context.store.list(scopeFilter, category, safeLimit, safeOffset); + const entries = await context.store.list(scopeFilter, category, safeLimit, safeOffset, { excludeInactive: !includeInvalidated }); if (entries.length === 0) { return { content: [{ type: "text", text: [ignoredScopeNotice, "No memories found."].filter(Boolean).join("\n") }], @@ -2152,9 +2153,10 @@ export function registerMemoryCompactTool(api, context) { scope: Type.Optional(Type.String({ description: "Optional scope filter." })), dryRun: Type.Optional(Type.Boolean({ description: "Preview compaction only (default true)." })), limit: Type.Optional(Type.Number({ description: "Max entries to scan (default 200)." })), + includeInvalidated: Type.Optional(Type.Boolean({ description: "Include invalidated/superseded rows in the scan (default false)." })), }), async execute(_toolCallId, params, _signal, _onUpdate, runtimeCtx) { - const { scope, dryRun = true, limit = 200 } = params; + const { scope, dryRun = true, limit = 200, includeInvalidated = false } = params; const safeLimit = clampInt(limit, 20, 1000); const agentId = resolveRuntimeAgentId(runtimeContext.agentId, runtimeCtx); let scopeFilter = resolveScopeFilter(context.scopeManager, agentId); @@ -2167,7 +2169,7 @@ export function registerMemoryCompactTool(api, context) { } scopeFilter = [scope]; } - const entries = await runtimeContext.store.list(scopeFilter, undefined, safeLimit, 0); + const entries = await runtimeContext.store.list(scopeFilter, undefined, safeLimit, 0, { excludeInactive: !includeInvalidated }); const canonicalByKey = new Map(); const duplicates = []; for (const entry of entries) { diff --git a/index.ts b/index.ts index 75efa8e8f..88a2ef51b 100644 --- a/index.ts +++ b/index.ts @@ -3218,6 +3218,7 @@ const memoryLanceDBProPlugin = { onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), migrator, embedder, + mdMirror, llmClient: smartExtractor ? (() => { try { const llmAuth = config.llm?.auth || "api-key"; @@ -5332,7 +5333,10 @@ const memoryLanceDBProPlugin = { } await mkdir(backupDir, { recursive: true }); - const allMemories = await store.list(undefined, undefined, 10000, 0); + // excludeInactive:false -- this is the automated backup dump and must + // keep full-dump semantics, including invalidated/superseded rows + // (item 6, PR #946). + const allMemories = await store.list(undefined, undefined, 10000, 0, { excludeInactive: false }); if (allMemories.length === 0) return; const dateStr = new Date().toISOString().split("T")[0]; diff --git a/package.json b/package.json index dfeb2daa5..9bac13fb0 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/memory-consolidate.test.mjs && node --test test/store-excludeinactive-default.test.mjs && node --test test/invalidated-rows-visibility.test.mjs && node --test test/memory-consolidate-cost-gate.test.mjs && node --test test/memory-consolidate-two-phase-apply.test.mjs && node --test test/memory-consolidate-admission-independence.test.mjs && node --test test/memory-consolidate-polish.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index c71de5313..29b98c0b9 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -1,35 +1,35 @@ -export const CI_TEST_GROUPS = [ - "cli-smoke", - "core-regression", - "storage-and-schema", - "llm-clients-and-auth", - "packaging-and-workflow", -]; - +export const CI_TEST_GROUPS = [ + "cli-smoke", + "core-regression", + "storage-and-schema", + "llm-clients-and-auth", + "packaging-and-workflow", +]; + export const CI_TEST_MANIFEST = [ { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" }, { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-max-input-chars.test.mjs", args: ["--test"] }, { group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-timestamp-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/storage-maintenance.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/read-consistency-interval.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/fts-index-fold.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-list-stats-projection-fallback.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/category-filter-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, - { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-timestamp-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/storage-maintenance.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/read-consistency-interval.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/fts-index-fold.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-list-stats-projection-fallback.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/category-filter-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, + { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, { group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/retriever-neighbor-enrichment.test.mjs", args: ["--test"] }, @@ -41,80 +41,87 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/dreaming-engine.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-governance-tools.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/corpus-indexer.test.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/openclaw-twitter-source-recipe.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/package-runtime.test.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/release-readiness.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, + { group: "packaging-and-workflow", runner: "node", file: "test/openclaw-twitter-source-recipe.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/package-runtime.test.mjs" }, + { group: "packaging-and-workflow", runner: "node", file: "test/release-readiness.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/self-improvement-reset-note.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/self-improvement.test.mjs", args: ["--test"] }, { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, { group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" }, { group: "core-regression", runner: "node", file: "test/memory-fact-query.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/redis-lock.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, - // Issue #598 regression tests - { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, - // Issue #629 batch embedding fix - { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, - // Issue #665 bulkStore tests - // Issue #690 cross-call batch accumulator tests - { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, - // Issue #665 bulkStore tests (from upstream) - { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-importance-normalization.test.mjs", args: ["--test"] }, - // Issue #680 regression tests (from upstream) - { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, - // Issue #606 SDK migration Bug 2 regression tests - { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, - // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback - { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, - // Issue #736 recall governance - isRecallUsed() unit tests - { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, - // Issue #492 agentId validation tests - { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, - // Tier 1 memory counter fix - { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/memory-subsession-prompt-hooks.test.mjs", args: ["--test"] }, - // Reflection distiller sub-session must not receive auto-recall/injected blocks - { group: "core-regression", runner: "node", file: "test/reflection-distiller-hook-skip.test.mjs", args: ["--test"] }, - // register() re-registration hardening (scope cache-miss log/handler dedup) + { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, + // Issue #598 regression tests + { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, + // Issue #629 batch embedding fix + { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, + // Issue #665 bulkStore tests + // Issue #690 cross-call batch accumulator tests + { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, + // Issue #665 bulkStore tests (from upstream) + { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-importance-normalization.test.mjs", args: ["--test"] }, + // Issue #680 regression tests (from upstream) + { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, + // Issue #606 SDK migration Bug 2 regression tests + { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, + // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback + { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, + // Issue #736 recall governance - isRecallUsed() unit tests + { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, + // Issue #492 agentId validation tests + { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, + // Tier 1 memory counter fix + { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-subsession-prompt-hooks.test.mjs", args: ["--test"] }, + // Reflection distiller sub-session must not receive auto-recall/injected blocks + { group: "core-regression", runner: "node", file: "test/reflection-distiller-hook-skip.test.mjs", args: ["--test"] }, + // register() re-registration hardening (scope cache-miss log/handler dedup) { group: "core-regression", runner: "node", file: "test/register-scope-dedup.test.mjs", args: ["--test"] }, - // Reflection distiller sub-run must request raw-run semantics (skip foreign before_prompt_build hooks) + // Reflection distiller sub-run must request raw-run semantics (skip foreign before_prompt_build hooks) { group: "core-regression", runner: "node", file: "test/raw-run-distiller-hooks.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/autocapture-watermark-reset.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/autocapture-internal-session-guard.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/memory-categories-storage-map.test.mjs", args: ["--test"] }, - // Delete/delete-bulk must synchronously invalidate in-process reflection read caches - { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, -]; - -export function getEntriesForGroup(group) { - if (!CI_TEST_GROUPS.includes(group)) { - throw new Error(`Unknown CI test group: ${group}`); - } - - return CI_TEST_MANIFEST.filter((entry) => entry.group === group); -} + { group: "core-regression", runner: "node", file: "test/autocapture-watermark-reset.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/autocapture-internal-session-guard.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/memory-categories-storage-map.test.mjs", args: ["--test"] }, + // Delete/delete-bulk must synchronously invalidate in-process reflection read caches + { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-excludeinactive-default.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-consolidate.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-consolidate-cost-gate.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-consolidate-two-phase-apply.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-consolidate-admission-independence.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-consolidate-polish.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/invalidated-rows-visibility.test.mjs", args: ["--test"] }, +]; + +export function getEntriesForGroup(group) { + if (!CI_TEST_GROUPS.includes(group)) { + throw new Error(`Unknown CI test group: ${group}`); + } + + return CI_TEST_MANIFEST.filter((entry) => entry.group === group); +} diff --git a/src/consolidate.ts b/src/consolidate.ts new file mode 100644 index 000000000..106f9b7f9 --- /dev/null +++ b/src/consolidate.ts @@ -0,0 +1,1166 @@ +import { createHash } from "node:crypto"; +import type { MemoryEntry } from "./store.js"; +import { + parseSmartMetadata, + buildSmartMetadata, + stringifySmartMetadata, + appendRelation, + deriveFactKey, + isMemoryActiveAt, + type SmartMemoryMetadata, +} from "./smart-metadata.js"; +import { APPEND_ONLY_CATEGORIES, type MemoryCategory } from "./memory-categories.js"; +import { + buildConsolidateBatchPrompt, + buildConsolidateBatchMergePrompt, + type ConsolidateBatchCluster, +} from "./extraction-prompts.js"; + +export type ConsolidateVerdict = "skip" | "merge" | "supersede" | "contradict"; + +export interface ConsolidateVerdictResult { + verdict: ConsolidateVerdict; + reason: string; + survivorIndex?: number; + absorbedIndices?: number[]; +} + +export interface ConsolidateCandidate { + entry: MemoryEntry; + memoryCategory?: MemoryCategory; + abstract: string; + overview: string; + content: string; + factKey?: string; + source?: string; + validFrom?: number; +} + +const REVERSAL_SIGNAL_PATTERN = + /\b(no longer|not anymore|any ?more|stopped|quit|used to|former|discontinued|doesn'?t|don'?t|isn'?t|wasn'?t)\b/i; + +const TOPIC_TOKEN_STOPWORDS = new Set([ + "user", "users", "prefer", "prefers", "preferred", "preference", "preferences", + "favorite", "favourite", "likes", "liked", "like", "dislikes", "dislike", + "drinking", "drinks", "drink", "drank", "still", "always", "anymore", "any", + "more", "longer", "stopped", "quit", "used", "no", "not", "the", "a", "an", + "of", "to", "and", "with", "their", "they", "was", "is", "are", "were", + "has", "have", "had", "will", "would", "their", "for", "at", "in", "on", + // Generic life-update narration: these appear across many unrelated life + // events/decisions and would otherwise let a single multi-topic narrative + // row bridge several unrelated topic clusters via incidental overlap. + "decided", "decide", "redesign", "redesigning", "relocate", + "relocating", "moved", "move", "moving", "changed", "change", "changing", + "switched", "switch", "started", "start", "starting", "continuing", + "continues", "testing", "tested", "experiment", "experimenting", "after", + "before", "now", "previously", "recently", "incident", "productivity", + "better", "correctly", "confirmed", "offered", "each", "record", "records", + "distinct", "fact", "facts", "note", "notes", "update", "updates", "updated", + "from", "into", "this", "that", "these", "those", "it", "its", "them", +]); + +function looksLikeReversal(text: string): boolean { + return REVERSAL_SIGNAL_PATTERN.test(text); +} + +function extractTopicTokens(text: string): Set { + const words = text.toLowerCase().match(/[a-z0-9][a-z0-9'-]{2,}/g) || []; + return new Set(words.filter((w) => !TOPIC_TOKEN_STOPWORDS.has(w))); +} + +// Reversal statements are typically short ("User will no longer drink +// cola"); a long multi-fact narrative recap can mention almost every topic +// in a scope at once and would otherwise bridge unrelated clusters through +// incidental keyword overlap. Only short, single-topic-looking statements +// participate in the topic-overlap fallback. +const REVERSAL_TOPIC_LINK_MAX_LENGTH = 120; + +function isEligibleForTopicLink(abstract: string): boolean { + return abstract.length <= REVERSAL_TOPIC_LINK_MAX_LENGTH; +} + +// Tokens match on exact equality or containment (one is a substring of the +// other, e.g. "cola" inside "coca-cola"), since brand/product names are +// routinely abbreviated across lanes. The shorter token must still be long +// enough (>= 4 chars) to keep an accidental short-token containment match +// from firing. +function tokensMatch(a: string, b: string): boolean { + if (a === b) return true; + const shorter = a.length <= b.length ? a : b; + const longer = a.length <= b.length ? b : a; + return shorter.length >= 4 && longer.includes(shorter); +} + +function shareSignificantTopicToken(a: string, b: string): boolean { + const tokensA = extractTopicTokens(a); + const tokensB = extractTopicTokens(b); + for (const tokenA of tokensA) { + for (const tokenB of tokensB) { + if (tokensMatch(tokenA, tokenB)) return true; + } + } + return false; +} + +// Fraction of the SMALLER topic-token set that matches the other set +// (0 when either side has no topic tokens at all). Two short statements +// about the same narrow fact typically share MOST of their significant +// words even when phrased completely differently across write lanes +// (e.g. "Favorite drink: cola" vs "Cola is what gets ordered most +// evenings" both reduce to essentially {"cola"}); two short statements +// about DIFFERENT facts rarely do, which is what keeps this fallback from +// bridging unrelated rows the way a single-shared-token check would. +function topicTokenOverlapRatio(a: string, b: string): number { + const tokensA = extractTopicTokens(a); + const tokensB = extractTopicTokens(b); + if (tokensA.size === 0 || tokensB.size === 0) return 0; + let matches = 0; + for (const tokenA of tokensA) { + for (const tokenB of tokensB) { + if (tokensMatch(tokenA, tokenB)) { + matches += 1; + break; + } + } + } + return matches / Math.min(tokensA.size, tokensB.size); +} + +const NEAR_DUPLICATE_TOKEN_OVERLAP_RATIO = 0.6; + +function cosineSimilarity(a: number[], b: number[]): number { + if (a.length === 0 || b.length === 0 || a.length !== b.length) return 0; + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA === 0 || normB === 0) return 0; + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +export function buildConsolidateCandidate(entry: MemoryEntry): ConsolidateCandidate { + const meta: SmartMemoryMetadata = parseSmartMetadata(entry.metadata, entry); + const abstract = meta.l0_abstract || entry.text; + const factKey = meta.fact_key || deriveFactKey(meta.memory_category, abstract); + return { + entry, + memoryCategory: meta.memory_category, + abstract, + overview: meta.l1_overview || "", + content: meta.l2_content || entry.text, + factKey, + source: meta.source, + validFrom: meta.valid_from, + }; +} + +function isDirectlyLinked( + a: ConsolidateCandidate, + b: ConsolidateCandidate, + similarityThreshold: number +): boolean { + const va = a.entry.vector; + const vb = b.entry.vector; + if (va.length > 0 && vb.length > 0 && cosineSimilarity(va, vb) >= similarityThreshold) { + return true; + } + if (a.factKey && a.factKey === b.factKey) { + return true; + } + // Reflection-mapped rows carry no stored fact_key, and a naturally phrased + // reversal rarely follows the "[Merge key]: text" convention that + // deriveFactKey needs to align across lanes, so its derived key is + // effectively unique. Gate a topic-word-overlap fallback to rows that look + // like a reversal, in the same category, and short enough to plausibly be + // about one topic, so it only widens linking for the exact case cosine + + // fact_key miss, not for arbitrary unrelated or multi-topic narrative rows. + if ( + (looksLikeReversal(a.abstract) || looksLikeReversal(b.abstract)) && + a.memoryCategory && + a.memoryCategory === b.memoryCategory && + isEligibleForTopicLink(a.abstract) && + isEligibleForTopicLink(b.abstract) && + shareSignificantTopicToken(a.abstract, b.abstract) + ) { + return true; + } + // Cross-lane near-duplicate fallback: two short, same-category rows that + // are NOT reversal-shaped can still be the same fact stated by different + // write lanes (manual/auto-capture/reflection*), whose differing + // tokenization keeps cosine and fact_key from matching. Unlike the + // reversal fallback above (which only needs ONE shared token, since a + // reversal is inherently pointed at a specific fact), this case requires + // a MAJORITY of the smaller row's topic tokens to overlap, so two short + // but topically different statements don't bridge on a single + // incidental shared word. + if ( + a.memoryCategory && + a.memoryCategory === b.memoryCategory && + isEligibleForTopicLink(a.abstract) && + isEligibleForTopicLink(b.abstract) && + topicTokenOverlapRatio(a.abstract, b.abstract) >= NEAR_DUPLICATE_TOKEN_OVERLAP_RATIO + ) { + return true; + } + return false; +} + +/** + * Seed-based clustering: for each not-yet-assigned row (in order), it + * becomes the seed of a new cluster, and every OTHER unassigned row joins + * that cluster only if it is DIRECTLY linked to the seed itself (cosine, + * fact_key, or the topic-overlap fallback) -- never transitively through + * another cluster member. Plain union-find (transitive closure) chains + * unrelated rows together whenever a series of only-moderately-similar + * pairs bridges them (row A links to B, B links to C, so A and C end up in + * one cluster even though A and C are never themselves similar); seed-based + * grouping caps that at a single hop from the seed, which is what keeps a + * handful of distinct topics from collapsing into one grab-bag cluster. + */ +export function clusterConsolidateCandidates( + candidates: ConsolidateCandidate[], + similarityThreshold: number +): number[][] { + const n = candidates.length; + const assigned = new Array(n).fill(false); + const clusters: number[][] = []; + + for (let seedIdx = 0; seedIdx < n; seedIdx++) { + if (assigned[seedIdx]) continue; + assigned[seedIdx] = true; + const cluster = [seedIdx]; + + for (let j = 0; j < n; j++) { + if (assigned[j]) continue; + if (isDirectlyLinked(candidates[seedIdx], candidates[j], similarityThreshold)) { + assigned[j] = true; + cluster.push(j); + } + } + + if (cluster.length >= 2) clusters.push(cluster); + } + + return clusters; +} + +export function chunkCluster(indices: number[], maxSize: number): number[][] { + const chunks: number[][] = []; + for (let i = 0; i < indices.length; i += maxSize) { + chunks.push(indices.slice(i, i + maxSize)); + } + return chunks; +} + +export function parseConsolidateVerdict(raw: unknown, memberCount: number): ConsolidateVerdictResult | null { + if (!raw || typeof raw !== "object") return null; + const obj = raw as Record; + const verdict = obj.verdict; + if (verdict !== "skip" && verdict !== "merge" && verdict !== "supersede" && verdict !== "contradict") { + return null; + } + const reason = typeof obj.reason === "string" ? obj.reason : ""; + + if (verdict === "skip" || verdict === "contradict") { + return { verdict, reason }; + } + + const survivorIndex = Number(obj.survivor_index); + if (!Number.isInteger(survivorIndex) || survivorIndex < 1 || survivorIndex > memberCount) return null; + + const absorbedRaw = obj.absorbed_indices; + if (!Array.isArray(absorbedRaw) || absorbedRaw.length === 0) return null; + const absorbedIndices = absorbedRaw.map((v) => Number(v)); + if ( + absorbedIndices.some( + (i) => !Number.isInteger(i) || i < 1 || i > memberCount || i === survivorIndex + ) + ) { + return null; + } + + return { verdict, reason, survivorIndex, absorbedIndices }; +} + +// Parses the batched decider's `{ verdicts: [...] }` response into a +// clusterIndex -> verdict map. Fails closed PER CLUSTER: an entry with an +// unrecognized/duplicate cluster_index, or a malformed verdict shape for its +// own member count, is simply dropped rather than discarding the whole +// batch -- callers treat a missing clusterIndex as "skip this cluster" the +// same way a single malformed per-cluster response was already handled. +export function parseConsolidateBatchVerdicts( + raw: unknown, + units: Array<{ clusterIndex: number; memberCount: number }> +): Map { + const result = new Map(); + if (!raw || typeof raw !== "object") return result; + + const verdictsRaw = (raw as Record).verdicts; + if (!Array.isArray(verdictsRaw)) return result; + + const memberCountByCluster = new Map(units.map((u) => [u.clusterIndex, u.memberCount])); + + for (const entry of verdictsRaw) { + if (!entry || typeof entry !== "object") continue; + const clusterIndex = Number((entry as Record).cluster_index); + if (!Number.isInteger(clusterIndex) || !memberCountByCluster.has(clusterIndex)) continue; + if (result.has(clusterIndex)) continue; + + const verdict = parseConsolidateVerdict(entry, memberCountByCluster.get(clusterIndex)!); + if (!verdict) continue; + + result.set(clusterIndex, verdict); + } + + return result; +} + +export interface ConsolidateAuditEntry { + action: "merge" | "supersede"; + survivorId: string; + absorbedIds: string[]; + reason: string; + scope: string; +} + +// ============================================================================ +// Item 7: LLM-cost gate. Clustering is free (local cosine + fact_key/topic +// linking); the only paid calls are the one batched decider call and one +// batched merge-content call (chunked past CONSOLIDATE_MERGE_BATCH_MAX_SIZE) +// covering every unit that MIGHT turn out to be a merge verdict, since item 8 +// moves merge-content generation into the plan phase. Both are knowable from +// clustering alone, before any LLM call is made -- which is what lets the +// gate sit ahead of the decide call and cover dry-runs as well as --apply. +// ============================================================================ + +/** Max merge jobs written in one batched merge-content LLM call; larger batches are chunked. */ +export const CONSOLIDATE_MERGE_BATCH_MAX_SIZE = 10; + +export interface ConsolidateCostPreview { + clusterCount: number; + /** Every unit might turn out to be a merge verdict: at most one merge job each. */ + maxMergeJobs: number; + /** ceil(maxMergeJobs / CONSOLIDATE_MERGE_BATCH_MAX_SIZE) batched merge-content calls. */ + maxMergeContentCalls: number; +} + +export function computeConsolidateCostPreview( + units: Array<{ members: unknown[] }> +): ConsolidateCostPreview { + const maxMergeJobs = units.length; + return { + clusterCount: units.length, + maxMergeJobs, + maxMergeContentCalls: Math.ceil(maxMergeJobs / CONSOLIDATE_MERGE_BATCH_MAX_SIZE), + }; +} + +export function pluralCount(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? "" : "s"}`; +} + +export function formatConsolidateCostPreview(preview: ConsolidateCostPreview): string { + const base = `${pluralCount(preview.clusterCount, "cluster")} -> 1 batched decider call`; + if (preview.maxMergeJobs === 0) return base; + return `${base} + worst case ${pluralCount(preview.maxMergeContentCalls, "batched merge-content call")} covering ${pluralCount(preview.maxMergeJobs, "merge job")}`; +} + +export function formatConsolidatePlanForDisplay(clusters: ClusterPlanReport[]): string { + const actionable = clusters.filter((c) => c.action); + const blocked = clusters.filter((c) => c.blocked === "append-only-shield"); + const noAction = clusters.filter((c) => !c.action && !c.blocked && !c.malformed && c.verdict); + if (actionable.length === 0 && blocked.length === 0 && noAction.length === 0) { + return "No actionable clusters in this plan."; + } + const lines: string[] = [ + `Plan: ${pluralCount(actionable.length, "actionable cluster")}, ${blocked.length} blocked, ${pluralCount(noAction.length, "skip")}`, + ]; + for (const cluster of actionable) { + lines.push(` [${cluster.action}] cluster ${cluster.clusterIndex} — ${cluster.verdict!.reason}`); + lines.push(` members: ${cluster.memberIds.join(", ")}`); + lines.push(` survivor: ${cluster.survivorId}`); + if (cluster.absorbedIds?.length) { + lines.push(` absorbed: ${cluster.absorbedIds.join(", ")}`); + } + if (cluster.action === "merge" && cluster.mergedContent) { + lines.push(` merged abstract: ${cluster.mergedContent.abstract}`); + lines.push(` merged overview: ${cluster.mergedContent.overview}`); + lines.push(` merged content: ${cluster.mergedContent.content}`); + } + } + for (const cluster of blocked) { + lines.push( + ` [${cluster.verdict!.verdict} — BLOCKED by append-only shield, will NOT be applied] cluster ${cluster.clusterIndex} — ${cluster.verdict!.reason}`, + ); + lines.push(` members: ${cluster.memberIds.join(", ")}`); + for (const text of cluster.memberTexts) { + lines.push(` - "${text}"`); + } + } + for (const cluster of noAction) { + lines.push(` [${cluster.verdict!.verdict}] cluster ${cluster.clusterIndex} — ${cluster.verdict!.reason}`); + for (const text of cluster.memberTexts) { + lines.push(` - "${text}"`); + } + } + return lines.join("\n"); +} + +// No `delete` method: no LLM verdict path may hard-delete a row. Both +// applyMergeVerdict and applySupersedeVerdict soft-invalidate absorbed rows +// via `update` only. Hard delete stays an operator-only CLI command, wired +// through a completely separate code path outside this pipeline. +export interface ConsolidateWriteDeps { + update: ( + id: string, + patch: { text?: string; vector?: number[]; metadata: string }, + scopeFilter?: string[] + ) => Promise; + embed: (text: string) => Promise; + completeJson: (prompt: string, label?: string, system?: string, temperature?: number) => Promise; +} + +export interface ConsolidateMergedContent { + abstract: string; + overview: string; + content: string; + vector: number[]; +} + +/** + * Item 8: pure content generation for merge verdicts -- one batched + * `consolidate-merge-batch` completion per chunk of up to + * CONSOLIDATE_MERGE_BATCH_MAX_SIZE merge verdicts (each job folds ALL of a + * verdict's absorbed members into its survivor in one output), plus one + * re-embed per job, with NO store writes. Called at PLAN-BUILD time (dry-run + * or --apply alike), so execution later can be pure store operations that + * never regenerate content and never call the LLM again. + * + * Per-item fail-closed: a response entry that is missing or malformed + * degrades ONLY that job to the survivor's own unmodified content -- exactly + * what the sequential per-member fold produced when its completions came + * back null -- and a chunk whose call itself fails degrades every job in + * that chunk the same way. Never throws, never fans back out into per-member + * LLM calls. + */ +async function buildMergePlanContentsBatch( + deps: Pick, + jobs: Array<{ members: ConsolidateCandidate[]; verdict: ConsolidateVerdictResult }>, + log?: (msg: string) => void +): Promise { + const out: ConsolidateMergedContent[] = new Array(jobs.length); + for (let chunkStart = 0; chunkStart < jobs.length; chunkStart += CONSOLIDATE_MERGE_BATCH_MAX_SIZE) { + const chunk = jobs.slice(chunkStart, chunkStart + CONSOLIDATE_MERGE_BATCH_MAX_SIZE); + const prompt = buildConsolidateBatchMergePrompt( + chunk.map(({ members, verdict }) => { + const survivor = members[verdict.survivorIndex! - 1]; + return { + category: survivor.memoryCategory || "preferences", + existing: { + abstract: survivor.abstract, + overview: survivor.overview, + content: survivor.content, + }, + additions: verdict.absorbedIndices!.map((idx) => { + const absorbed = members[idx - 1]; + return { + abstract: absorbed.abstract, + overview: absorbed.overview, + content: absorbed.content, + }; + }), + }; + }) + ); + + const byIndex = new Map(); + try { + const raw = await deps.completeJson<{ + results?: Array<{ index?: number; abstract?: string; overview?: string; content?: string }>; + }>(prompt.user, "consolidate-merge-batch", prompt.system); + for (const entry of raw && Array.isArray(raw.results) ? raw.results : []) { + if (!entry || typeof entry.index !== "number") continue; + byIndex.set(entry.index, entry); + } + } catch (err) { + log?.( + `memory-consolidate: batched merge-content call failed, keeping survivor content for ${chunk.length} job(s): ${String(err)}` + ); + } + + for (let i = 0; i < chunk.length; i++) { + const { members, verdict } = chunk[i]; + const survivor = members[verdict.survivorIndex! - 1]; + const entry = byIndex.get(i + 1); + const usable = + entry && + typeof entry.abstract === "string" && + entry.abstract.trim().length > 0 && + typeof entry.overview === "string" && + typeof entry.content === "string"; + if (!usable) { + log?.( + "memory-consolidate: missing or malformed merge-content entry, keeping survivor content for this job" + ); + } + const abstract = usable ? (entry!.abstract as string) : survivor.abstract; + const overview = usable ? (entry!.overview as string) : survivor.overview; + const content = usable ? (entry!.content as string) : survivor.content; + const vector = await deps.embed(`${abstract} ${content}`); + out[chunkStart + i] = { abstract, overview, content, vector }; + } + } + return out; +} + +/** + * Item 8: pure store write for an already-planned merge verdict. Applies + * EXACTLY the precomputed content from `buildMergePlanContentsBatch` -- no LLM + * call, no regeneration, "apply exactly what was presented." + */ +async function writeMergeVerdict( + deps: Pick, + members: ConsolidateCandidate[], + verdict: ConsolidateVerdictResult, + mergedContent: ConsolidateMergedContent, + scopeFilter: string[] | undefined, + now: number +): Promise { + const survivor = members[verdict.survivorIndex! - 1]; + const { abstract, overview, content, vector } = mergedContent; + + const absorbedIds: string[] = []; + for (const idx of verdict.absorbedIndices!) { + absorbedIds.push(members[idx - 1].entry.id); + } + + const patchedMeta = buildSmartMetadata(survivor.entry, { + l0_abstract: abstract, + l1_overview: overview, + l2_content: content, + }); + const auditedMeta = { + ...patchedMeta, + consolidation_audit: { action: "merge", absorbedIds, reason: verdict.reason, at: now }, + }; + await deps.update( + survivor.entry.id, + { text: abstract, vector, metadata: stringifySmartMetadata(auditedMeta) }, + scopeFilter + ); + + // Non-destructive: absorbed rows are soft-invalidated with the same + // primitive applySupersedeVerdict uses (invalidated_at + superseded_by + + // relations), not hard-deleted. Each absorbed row also gets its own + // consolidation_audit pointing back at the survivor, so its history is + // independently inspectable without cross-referencing the survivor's + // audit. No LLM verdict path may call a hard delete; hard delete stays an + // operator-only CLI command. + for (const idx of verdict.absorbedIndices!) { + const absorbed = members[idx - 1]; + const existingMeta = parseSmartMetadata(absorbed.entry.metadata, absorbed.entry); + const invalidatedMeta = buildSmartMetadata(absorbed.entry, { + invalidated_at: now, + superseded_by: survivor.entry.id, + relations: appendRelation(existingMeta.relations, { type: "superseded_by", targetId: survivor.entry.id }), + }); + const auditedAbsorbedMeta = { + ...invalidatedMeta, + consolidation_audit: { action: "merge", survivorId: survivor.entry.id, reason: verdict.reason, at: now }, + }; + await deps.update(absorbed.entry.id, { metadata: stringifySmartMetadata(auditedAbsorbedMeta) }, scopeFilter); + } + + return { action: "merge", survivorId: survivor.entry.id, absorbedIds, reason: verdict.reason, scope: survivor.entry.scope }; +} + +async function applySupersedeVerdict( + deps: Pick, + members: ConsolidateCandidate[], + verdict: ConsolidateVerdictResult, + scopeFilter: string[] | undefined, + now: number +): Promise { + const survivor = members[verdict.survivorIndex! - 1]; + const factKey = survivor.factKey || members[verdict.absorbedIndices![0] - 1].factKey || ""; + const absorbedIds: string[] = []; + + for (const idx of verdict.absorbedIndices!) { + const absorbed = members[idx - 1]; + const existingMeta = parseSmartMetadata(absorbed.entry.metadata, absorbed.entry); + const invalidatedMetadata = buildSmartMetadata(absorbed.entry, { + fact_key: factKey || existingMeta.fact_key, + invalidated_at: now, + superseded_by: survivor.entry.id, + relations: appendRelation(existingMeta.relations, { type: "superseded_by", targetId: survivor.entry.id }), + }); + await deps.update(absorbed.entry.id, { metadata: stringifySmartMetadata(invalidatedMetadata) }, scopeFilter); + absorbedIds.push(absorbed.entry.id); + } + + // An append-only (events/cases) survivor is left byte-untouched: the shield + // admits it only because nothing gets written to it, so even the fact_key + // patch and audit annotation are skipped (the audit still lands in the plan + // report and journal mirror). + const survivorIsAppendOnly = Boolean( + survivor.memoryCategory && APPEND_ONLY_CATEGORIES.has(survivor.memoryCategory), + ); + if (!survivorIsAppendOnly) { + const survivorMeta = parseSmartMetadata(survivor.entry.metadata, survivor.entry); + const patchedSurvivorMeta = buildSmartMetadata(survivor.entry, { + fact_key: factKey || survivorMeta.fact_key, + }); + const auditedMeta = { + ...patchedSurvivorMeta, + consolidation_audit: { action: "supersede", absorbedIds, reason: verdict.reason, at: now }, + }; + await deps.update(survivor.entry.id, { metadata: stringifySmartMetadata(auditedMeta) }, scopeFilter); + } + + return { action: "supersede", survivorId: survivor.entry.id, absorbedIds, reason: verdict.reason, scope: survivor.entry.scope }; +} + +export interface ClusterPlanReport { + clusterIndex: number; + memberIds: string[]; + memberTexts: string[]; + verdict: ConsolidateVerdictResult | null; + malformed: boolean; + /** + * Why an undecided cluster is undecided: the whole decide call returned + * nothing (provider error/timeout) vs. the call succeeded but this + * cluster's verdict was missing or unparseable. Only set when malformed. + */ + failure?: "call-failed" | "malformed-verdict"; + /** Set when a decided verdict was withheld by the append-only shield. */ + blocked?: "append-only-shield"; + /** Stable identity of this cluster's member set + content, for the settled ledger. */ + fingerprint?: string; + /** null for skip/contradict/malformed/append-only-blocked units -- nothing to execute. */ + action: "merge" | "supersede" | null; + survivorId?: string; + absorbedIds?: string[]; + /** Item 8: precomputed at plan-build time, applied verbatim at execution. */ + mergedContent?: ConsolidateMergedContent; + /** Snapshot used by the item-8 staleness guard: each member's id + exact metadata string at plan-build time. */ + staleness: Array<{ id: string; metadata: string | undefined }>; +} + +export interface RunConsolidateOptions { + scope: string; + scopeFilter?: string[]; + category?: MemoryCategory; + sinceMs?: number; + includeReflectionSlices?: boolean; + similarityThreshold?: number; + clusterCap?: number; + apply: boolean; + now?: number; + /** --yes: bypasses the item-7 cost gate without ever calling confirmCost. */ + autoConfirm?: boolean; + /** + * Fingerprints of clusters settled by previous runs (skip/contradict + * verdicts and shield-blocked verdicts). Matching clusters are dropped + * before the cost gate and never reach the decider, so repeated runs + * converge to zero clusters. A fingerprint covers each member's exact + * metadata, so any member change re-opens its cluster automatically. + */ + settledFingerprints?: Set; +} + +export interface RunConsolidateDeps extends ConsolidateWriteDeps { + fetchRows: (scopeFilter: string[] | undefined, maxTimestamp: number, limit: number) => Promise; + /** Re-fetches a row by id for the item-8 staleness guard. Omit to skip the guard (all clusters treated as fresh). */ + getById?: (id: string, scopeFilter?: string[]) => Promise; + /** + * Item 7 cost gate: called with a preview message before any LLM call, + * unless options.autoConfirm is set. A declined or missing confirmCost + * (and !autoConfirm) is a safe abort -- fail closed, never assume consent. + */ + confirmCost?: (message: string) => Promise; + /** + * Item 8 apply gate: called with the fully-built plan (message + per- + * cluster detail) when options.apply is false, so the user can review + * before anything is written. Never called when options.apply is true + * (direct --apply executes immediately, no second prompt). A declined or + * missing confirmApply is a safe no-op -- nothing gets written. + */ + confirmApply?: (message: string, clusters: ClusterPlanReport[]) => Promise; + onAudit?: (audit: ConsolidateAuditEntry) => Promise | void; + log?: (message: string) => void; +} + +export interface RunConsolidateResult { + /** "aborted": the item-7 cost gate was declined (or unavailable) -- zero LLM calls were made. */ + status: "aborted" | "completed"; + abortReason?: string; + scanned: number; + eligible: number; + costPreview?: ConsolidateCostPreview; + clusters: ClusterPlanReport[]; + applied: ConsolidateAuditEntry[]; + /** True iff the plan (or the fresh subset of it) was actually written to the store. */ + executed: boolean; + /** Clusters withheld at execution time because a member row changed or disappeared since the plan was built. */ + staleSkipped: Array<{ clusterIndex: number; memberIds: string[] }>; + /** Clusters whose verdict was missing/unparseable while the decide call itself succeeded. */ + skippedMalformed: number; + /** Clusters left undecided because the decide call returned no response at all. */ + undecidedCallFailed: number; + /** Clusters dropped before the decider because a previous run already settled them. */ + settledSkipped: number; + /** Fingerprints newly settled by this run (skip/contradict/shield-blocked outcomes). */ + newlySettled: string[]; + apply: boolean; +} + +const DEFAULT_SIMILARITY_THRESHOLD = 0.86; +const DEFAULT_CLUSTER_CAP = 8; +const DEFAULT_SCAN_LIMIT = 100_000; + +function abortedResult( + reason: string, + scanned: number, + eligible: number, + costPreview: ConsolidateCostPreview | undefined, + apply: boolean +): RunConsolidateResult { + return { + status: "aborted", + abortReason: reason, + scanned, + eligible, + costPreview, + clusters: [], + applied: [], + executed: false, + staleSkipped: [], + skippedMalformed: 0, + undecidedCallFailed: 0, + settledSkipped: 0, + newlySettled: [], + apply, + }; +} + +/** + * Stable identity for a cluster in the settled ledger: the sorted member + * ids with each member's exact metadata string. Any member change (edit, + * merge, invalidation) or any membership change produces a different + * fingerprint, so a settled entry can never suppress a cluster whose + * content moved on. + */ +export function computeClusterFingerprint( + members: Array<{ id: string; metadata: string | undefined }>, +): string { + const parts = members + .map((m) => `${m.id}\n${m.metadata ?? ""}`) + .sort() + .join(""); + return createHash("sha256").update(parts).digest("hex"); +} + +/** + * Item 8 staleness guard: re-fetches every member of a plan entry and + * compares its metadata string against the plan-build-time snapshot. + * Missing row (disappeared) or changed metadata (mutated by someone else) + * both count as stale. Skips the check entirely (treats as fresh) when + * deps.getById isn't provided -- an opt-in safety net, not a hard + * requirement, so callers that don't need it don't have to wire it up. + */ +async function isClusterFresh( + deps: Pick, + entry: ClusterPlanReport, + scopeFilter: string[] | undefined +): Promise { + if (!deps.getById) return true; + for (const snapshot of entry.staleness) { + const current = await deps.getById(snapshot.id, scopeFilter); + if (!current) return false; + if (current.metadata !== snapshot.metadata) return false; + } + return true; +} + +async function executePlan( + deps: RunConsolidateDeps, + clusters: ClusterPlanReport[], + membersByCluster: Map, + scopeFilter: string[] | undefined, + now: number +): Promise<{ applied: ConsolidateAuditEntry[]; staleSkipped: Array<{ clusterIndex: number; memberIds: string[] }> }> { + const applied: ConsolidateAuditEntry[] = []; + const staleSkipped: Array<{ clusterIndex: number; memberIds: string[] }> = []; + + for (const entry of clusters) { + if (!entry.action || !entry.verdict) continue; + const members = membersByCluster.get(entry.clusterIndex); + if (!members) continue; + + const fresh = await isClusterFresh(deps, entry, scopeFilter); + if (!fresh) { + staleSkipped.push({ clusterIndex: entry.clusterIndex, memberIds: entry.memberIds }); + deps.log?.( + `memory-consolidate: cluster ${entry.clusterIndex} changed since the plan was built (stale); skipping, never partially applied` + ); + continue; + } + + try { + const audit = + entry.action === "merge" + ? await writeMergeVerdict(deps, members, entry.verdict, entry.mergedContent!, scopeFilter, now) + : await applySupersedeVerdict(deps, members, entry.verdict, scopeFilter, now); + applied.push(audit); + await deps.onAudit?.(audit); + } catch (err) { + deps.log?.(`memory-consolidate: failed to apply ${entry.action} verdict: ${String(err)}`); + } + } + + return { applied, staleSkipped }; +} + +export async function runConsolidate( + deps: RunConsolidateDeps, + options: RunConsolidateOptions +): Promise { + const now = options.now ?? Date.now(); + const scopeFilter = options.scopeFilter ?? [options.scope]; + + const rawEntries = await deps.fetchRows(scopeFilter, now, DEFAULT_SCAN_LIMIT); + + const filtered = rawEntries.filter((entry) => { + if (entry.category === "reflection" && !options.includeReflectionSlices) return false; + if (options.sinceMs !== undefined && entry.timestamp < options.sinceMs) return false; + return true; + }); + + const candidates = filtered + .map(buildConsolidateCandidate) + .filter((candidate) => { + const meta = parseSmartMetadata(candidate.entry.metadata, candidate.entry); + if (!isMemoryActiveAt(meta, now)) return false; + if (options.category && candidate.memoryCategory !== options.category) return false; + return true; + }) + // Sort by row id (a stable key) before clustering, not just before + // building the prompt: clusterConsolidateCandidates' seed-based scan + // always picks the lowest surviving array index as the next seed, so a + // pre-sorted candidate array makes both which rows end up in the same + // cluster AND their order within it a pure function of the candidate + // SET -- independent of whatever order fetchRows happened to return + // this call, which store/DB internals don't guarantee is stable. + .sort((a, b) => (a.entry.id < b.entry.id ? -1 : a.entry.id > b.entry.id ? 1 : 0)); + + const similarityThreshold = options.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD; + const clusterCap = options.clusterCap ?? DEFAULT_CLUSTER_CAP; + const clusterIndexGroups = clusterConsolidateCandidates(candidates, similarityThreshold); + + const byId = (a: ConsolidateCandidate, b: ConsolidateCandidate) => + a.entry.id < b.entry.id ? -1 : a.entry.id > b.entry.id ? 1 : 0; + + // Flatten every cluster (and any cluster chunked past clusterCap) into a + // single ordered list of decision units first, so the decider can be + // asked about all of them in ONE completeJson call instead of one call + // per cluster. Members within a unit, and units themselves, are + // explicitly re-sorted by row id here too (belt-and-suspenders on top of + // the pre-clustering sort above) so prompt assembly never depends on + // clusterConsolidateCandidates' internal grouping order. + const units: Array<{ clusterIndex: number; members: ConsolidateCandidate[] }> = []; + for (const group of clusterIndexGroups) { + const sortedGroup = [...group].sort((a, b) => byId(candidates[a], candidates[b])); + const chunks = chunkCluster(sortedGroup, clusterCap); + for (const chunkIndices of chunks) { + if (chunkIndices.length < 2) continue; + units.push({ clusterIndex: units.length + 1, members: chunkIndices.map((i) => candidates[i]) }); + } + } + units.sort((a, b) => byId(a.members[0], b.members[0])); + + // Convergence: clusters settled by a previous run (same members, same + // content) are dropped before the cost gate and the decider ever see + // them, so repeated runs over an unchanged store reach zero clusters. + const fingerprintByUnit = new Map<(typeof units)[number], string>(); + for (const unit of units) { + fingerprintByUnit.set( + unit, + computeClusterFingerprint(unit.members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata }))), + ); + } + let settledSkipped = 0; + const activeUnits = units.filter((unit) => { + if (options.settledFingerprints?.has(fingerprintByUnit.get(unit)!)) { + settledSkipped += 1; + return false; + } + return true; + }); + units.length = 0; + units.push(...activeUnits); + units.forEach((unit, i) => { + unit.clusterIndex = i + 1; + }); + + // Item 7: the cost gate sits here -- clustering above is free (local + // cosine + fact_key/topic linking), and everything below this point is + // the first LLM call onward. Skipped entirely when there's nothing to + // decide (nothing to confirm), and bypassed without ever calling + // confirmCost when autoConfirm (--yes) is set. A declined OR missing + // confirmCost is treated identically: a safe abort, never assumed consent. + let costPreview: ConsolidateCostPreview | undefined; + if (units.length > 0) { + costPreview = computeConsolidateCostPreview(units); + if (!options.autoConfirm) { + const message = formatConsolidateCostPreview(costPreview); + const proceed = deps.confirmCost ? await deps.confirmCost(message) : false; + if (!proceed) { + return abortedResult( + "cost gate declined (or no confirmCost dep and --yes not set): no LLM call was made", + rawEntries.length, + candidates.length, + costPreview, + options.apply + ); + } + } + } + + const clusters: ClusterPlanReport[] = []; + const membersByCluster = new Map(); + const pendingMergeContent: Array<{ + cluster: ClusterPlanReport; + members: ConsolidateCandidate[]; + verdict: ConsolidateVerdictResult; + }> = []; + let skippedMalformed = 0; + let undecidedCallFailed = 0; + let decideCallFailed = false; + const newlySettled: string[] = []; + + if (units.length > 0) { + const batchClusters: ConsolidateBatchCluster[] = units.map((unit) => ({ + clusterIndex: unit.clusterIndex, + members: unit.members.map((m, i) => ({ + index: i + 1, + category: m.memoryCategory || "preferences", + abstract: m.abstract, + overview: m.overview, + content: m.content, + source: m.source, + timestamp: m.entry.timestamp, + validFrom: m.validFrom, + })), + })); + const prompt = buildConsolidateBatchPrompt(batchClusters); + const raw = await deps.completeJson>(prompt.user, "consolidate-decide", prompt.system, 0); + decideCallFailed = raw === null || raw === undefined; + if (decideCallFailed) { + deps.log?.( + `memory-consolidate: consolidate-decide call returned no response (provider error or timeout); ${units.length} cluster(s) left undecided` + ); + } + const verdictMap = !decideCallFailed + ? parseConsolidateBatchVerdicts( + raw!, + units.map((u) => ({ clusterIndex: u.clusterIndex, memberCount: u.members.length })) + ) + : new Map(); + + // Build the COMPLETE plan now, regardless of apply/dry-run -- + // every merge verdict gets its content generated here (moved from + // apply time), so execution later is pure store writes with zero + // further LLM calls. + for (const unit of units) { + const members = unit.members; + const verdict = verdictMap.get(unit.clusterIndex) ?? null; + const fingerprint = fingerprintByUnit.get(unit)!; + membersByCluster.set(unit.clusterIndex, members); + + if (!verdict) { + if (decideCallFailed) { + undecidedCallFailed += 1; + } else { + skippedMalformed += 1; + deps.log?.( + `memory-consolidate: missing or malformed verdict for a cluster of ${members.length} rows, skipping` + ); + } + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict: null, + malformed: true, + failure: decideCallFailed ? "call-failed" : "malformed-verdict", + fingerprint, + action: null, + staleness: members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata })), + }); + continue; + } + + const staleness = members.map((m) => ({ id: m.entry.id, metadata: m.entry.metadata })); + + if (verdict.verdict === "skip" || verdict.verdict === "contradict") { + newlySettled.push(fingerprint); + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + fingerprint, + action: null, + staleness, + }); + continue; + } + + const survivorCategory = members[verdict.survivorIndex! - 1].memoryCategory; + const absorbedCategories = verdict.absorbedIndices!.map((idx) => members[idx - 1].memoryCategory); + const survivorIsAppendOnly = Boolean(survivorCategory && APPEND_ONLY_CATEGORIES.has(survivorCategory)); + const absorbedTouchesAppendOnly = absorbedCategories.some( + (category) => category && APPEND_ONLY_CATEGORIES.has(category), + ); + // Append-only means invalidation-protection, not merge-immunity, and + // the protection is directional: absorbed rows are what get invalidated + // (and merge additionally rewrites the survivor's content), so an + // append-only row may never be absorbed, and may only be a merge + // survivor when every acted-upon row shares the identical append-only + // category (a genuine same-category duplicate). A supersede survivor is + // never written at all when it is append-only (applySupersedeVerdict + // skips even the audit annotation), so an append-only row superseding + // stale mutable rows leaves the append-only guarantee intact. + const isSameCategoryAppendOnlyMerge = + verdict.verdict === "merge" && + survivorIsAppendOnly && + absorbedCategories.every((category) => category === survivorCategory); + const blockedByShield = + verdict.verdict === "merge" + ? (survivorIsAppendOnly || absorbedTouchesAppendOnly) && !isSameCategoryAppendOnlyMerge + : absorbedTouchesAppendOnly; + if (blockedByShield) { + deps.log?.( + `memory-consolidate: refusing to ${verdict.verdict} an append-only row (events/cases) outside a same-category duplicate merge; skipping this verdict` + ); + // A shield-blocked verdict is as settled as a skip: re-running the + // decider over the same unchanged members can only produce another + // blocked verdict. + newlySettled.push(fingerprint); + clusters.push({ + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + blocked: "append-only-shield", + fingerprint, + action: null, + staleness, + }); + continue; + } + + const survivor = members[verdict.survivorIndex! - 1]; + const absorbedIds = verdict.absorbedIndices!.map((idx) => members[idx - 1].entry.id); + + const cluster: ClusterPlanReport = { + clusterIndex: unit.clusterIndex, + memberIds: members.map((m) => m.entry.id), + memberTexts: members.map((m) => m.abstract), + verdict, + malformed: false, + fingerprint, + action: verdict.verdict === "merge" ? "merge" : "supersede", + survivorId: survivor.entry.id, + absorbedIds, + staleness, + }; + clusters.push(cluster); + if (verdict.verdict === "merge") { + pendingMergeContent.push({ cluster, members, verdict }); + } + } + + // One batched merge-content call (chunk-capped) covers every merge + // verdict's plan content, moved out of the per-unit loop so the plan + // build spends ceil(M/CONSOLIDATE_MERGE_BATCH_MAX_SIZE) LLM calls + // instead of one call per absorbed member. + if (pendingMergeContent.length > 0) { + const contents = await buildMergePlanContentsBatch(deps, pendingMergeContent, deps.log); + pendingMergeContent.forEach((pending, i) => { + pending.cluster.mergedContent = contents[i]; + }); + } + } + + const actionable = clusters.filter((c) => c.action); + + // Item 8: direct --apply executes the plan immediately, no second prompt. + if (options.apply) { + const { applied, staleSkipped } = await executePlan(deps, actionable, membersByCluster, scopeFilter, now); + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied, + executed: true, + staleSkipped, + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: true, + }; + } + + // Dry-run / interactive path: present the full plan, ask once, execute + // only on an explicit affirmative. A declined or missing confirmApply is + // a safe no-op -- the plan was built (and its LLM calls already spent), + // but nothing is written. + if (actionable.length > 0) { + const message = `${pluralCount(actionable.length, "cluster")} ready to apply. Apply these now? (YES/no)`; + const proceed = deps.confirmApply ? await deps.confirmApply(message, clusters) : false; + if (proceed) { + const { applied, staleSkipped } = await executePlan(deps, actionable, membersByCluster, scopeFilter, now); + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied, + executed: true, + staleSkipped, + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: false, + }; + } + } + + return { + status: "completed", + scanned: rawEntries.length, + eligible: candidates.length, + costPreview, + clusters, + applied: [], + executed: false, + staleSkipped: [], + skippedMalformed, + undecidedCallFailed, + settledSkipped, + newlySettled, + apply: false, + }; +} diff --git a/src/extraction-prompts.ts b/src/extraction-prompts.ts index 93a787f4c..4d934594c 100644 --- a/src/extraction-prompts.ts +++ b/src/extraction-prompts.ts @@ -220,3 +220,228 @@ Return JSON: "content": "Merged full content" } `; } + +export interface SplitPrompt { + system: string; + user: string; +} + +export interface ConsolidateMember { + index: number; + category: string; + abstract: string; + overview: string; + content: string; + source?: string; + timestamp?: number; + validFrom?: number; +} + +export const CONSOLIDATE_MERGE_SYSTEM_PROMPT = `You are a memory consolidation merge writer. Merge two versions of the same memory into a single coherent record with all three levels (abstract, overview, content). + +Requirements: +- Remove duplicate information +- Keep the most up-to-date details +- Maintain a coherent narrative +- Keep code identifiers, URIs, and model names unchanged when they are proper nouns + +Return JSON only: +{ + "abstract": "Merged one-line abstract", + "overview": "Merged structured Markdown overview", + "content": "Merged full content" +}`; + +// mapped/manual/legacy rows without a real overview/content commonly fall +// back to the raw abstract text in all three tiers (see +// src/smart-metadata.ts's parseSmartMetadata: l2_content falls back to raw +// text, l1_overview falls back to `- ${abstract}`). Printing that fact three +// times per member wastes cluster-listing space for no signal. +function hasThinTiers(m: ConsolidateMember): boolean { + const overviewIsDefault = m.overview === "" || m.overview === `- ${m.abstract}` || m.overview === m.abstract; + const contentIsDefault = m.content === m.abstract; + return overviewIsDefault && contentIsDefault; +} + +function formatMemberHeader(m: ConsolidateMember): string { + const parts = [`${m.index}. [${m.category}]`]; + if (m.source) parts.push(` (source: ${m.source})`); + if (m.timestamp !== undefined) { + parts.push(`, timestamp: ${new Date(m.timestamp).toISOString()}`); + if (m.validFrom !== undefined && m.validFrom !== m.timestamp) { + parts.push(`, valid_from: ${new Date(m.validFrom).toISOString()}`); + } + } + return parts.join(""); +} + +function formatMemberTiers(m: ConsolidateMember): string { + if (hasThinTiers(m)) { + return `Fact: ${m.abstract}`; + } + return `Abstract: ${m.abstract}\nOverview: ${m.overview}\nContent: ${m.content}`; +} + +export function buildConsolidatePrompt(members: ConsolidateMember[]): SplitPrompt { + const system = `You are a memory consolidation decider. You are given a cluster of existing memories that were flagged as likely related, either by embedding similarity or by sharing a topic key. Decide how to reconcile the ACTIONABLE rows in this cluster. You do NOT have to act on every row: survivor_index and absorbed_indices only need to cover the rows you are deciding about. Any row you leave out of both is simply left untouched — this is expected and correct whenever a cluster mixes actionable duplicates or reversals with unrelated or append-only rows. + +Return exactly one verdict, scoped to whichever rows it actually applies to: +- skip: none of the rows in this cluster need any action. Use this only when nothing here is a duplicate, reversal, or contradiction. +- merge: two or more rows are duplicates or near-duplicates of the same fact. Pick the row with the best-quality, most complete text as the survivor and list only the true duplicates as absorbed. +- supersede: one row is a newer fact or an explicit reversal that replaces one or more older rows describing the same fact (for example, a decision to stop doing something an older row describes). The survivor is the newer/reversal row; list only the rows it actually replaces as absorbed. Supersede is NOT destructive: absorbed rows are never deleted. They are kept as an auditable historical record and simply marked as no longer current, exactly like SUPERSEDE in ordinary dedup decisions ("the same mutable fact has changed over time; keep the old memory as historical but no longer current"). Use supersede whenever a row states that a fact from an older row has changed, even if that only applies to part of the cluster. +- contradict: two or more rows conflict and it is not clear which one is correct. Flag this for human review. No destructive action. + +"events" and "cases" categories are append-only: they can never be superseded or contradicted (append-only means invalidation-protection, not merge-immunity). A merge must never mix an append-only row with a non-append-only row, or with a different append-only category. The one exception: near-identical duplicate rows within the SAME append-only category (for example two "events" rows describing the exact same occurrence, or two "cases" rows describing the exact same problem/solution) may still be merged like any other true duplicate. Outside that same-category duplicate case, leave append-only rows out of absorbed_indices, with one directional exception: an append-only row MAY serve as the supersede survivor_index when every absorbed row is non-append-only — the append-only row itself is never written, only the stale mutable rows get marked no longer current. None of this ever blocks you from merging or superseding the OTHER, actionable rows in the same cluster. + +Rows in DIFFERENT non-append-only categories (profile, preferences, entities, patterns) are fully actionable against each other — differing categories alone are never a reason to skip. Merge them when they state the same fact, choosing the more authoritative category's row as survivor (for identity facts like the user's name, profile over preferences); supersede when they conflict about the same fact, choosing the factually current row as survivor. Factual currency always decides supersede direction: never make a stale row the survivor for category reasons, and when the stale side is append-only (so it cannot be absorbed), use skip rather than a wrong-direction supersede. + +Source legend: legacy = pre-smart-format rows, manual = operator memory_store saves, auto-capture = extraction lane, reflection* = mirror lanes; manual rows are operator-authored and strong survivor candidates. + +Each member below also shows its timestamp (and valid_from when it differs) — use these to judge supersede recency explicitly rather than inferring it from wording alone. + +Return JSON only: +{ + "verdict": "skip|merge|supersede|contradict", + "survivor_index": 1, + "absorbed_indices": [2, 3], + "reason": "short explanation" +} + +Only include survivor_index and absorbed_indices for merge or supersede. survivor_index and every entry in absorbed_indices must be one of the row numbers shown below. absorbed_indices must never contain an append-only (events/cases) row — unless the verdict is merge and every row in survivor_index/absorbed_indices shares the exact same append-only category. An append-only row may appear as survivor_index only for that same-category duplicate merge, or for a supersede whose absorbed rows are all non-append-only.`; + + const user = `Cluster members:\n\n${members + .map((m) => `${formatMemberHeader(m)}\n${formatMemberTiers(m)}`) + .join("\n\n")}`; + + return { system, user }; +} + +export interface ConsolidateBatchCluster { + clusterIndex: number; + members: ConsolidateMember[]; +} + +// Same decider semantics as buildConsolidatePrompt, but scoped to decide +// N independent clusters in a single call: one LLM round-trip per +// consolidate run instead of one per cluster. Each cluster is decided +// independently -- a verdict for one cluster must never be influenced by +// another cluster's rows -- and the response is a JSON array with one +// verdict object per cluster, tagged by cluster_index so a malformed entry +// for one cluster can be dropped without discarding the others' verdicts. +export function buildConsolidateBatchPrompt(clusters: ConsolidateBatchCluster[]): SplitPrompt { + const system = `You are a memory consolidation decider. You are given multiple independent clusters of existing memories, each flagged as likely related within itself, either by embedding similarity or by sharing a topic key. Decide how to reconcile the ACTIONABLE rows in EACH cluster independently -- a decision about one cluster must never be influenced by another cluster's rows. You do NOT have to act on every row in a cluster: survivor_index and absorbed_indices only need to cover the rows you are deciding about within that cluster. Any row you leave out of both is simply left untouched -- this is expected and correct whenever a cluster mixes actionable duplicates or reversals with unrelated or append-only rows. + +Return exactly one verdict per cluster, scoped to whichever rows it actually applies to: +- skip: none of the rows in this cluster need any action. Use this only when nothing here is a duplicate, reversal, or contradiction. +- merge: two or more rows are duplicates or near-duplicates of the same fact. Pick the row with the best-quality, most complete text as the survivor and list only the true duplicates as absorbed. +- supersede: one row is a newer fact or an explicit reversal that replaces one or more older rows describing the same fact (for example, a decision to stop doing something an older row describes). The survivor is the newer/reversal row; list only the rows it actually replaces as absorbed. Supersede is NOT destructive: absorbed rows are never deleted. They are kept as an auditable historical record and simply marked as no longer current, exactly like SUPERSEDE in ordinary dedup decisions ("the same mutable fact has changed over time; keep the old memory as historical but no longer current"). Use supersede whenever a row states that a fact from an older row has changed, even if that only applies to part of the cluster. +- contradict: two or more rows conflict and it is not clear which one is correct. Flag this for human review. No destructive action. + +Decision criteria: apply these checks in order for the rows in each cluster. +1. Do two or more rows say the same thing, with no row stating a newer fact, a change, or a reversal? -> merge. +2. Does one row explicitly state a fact has changed, ended, or reversed relative to another row (wording like "no longer", "stopped", "switched to", or simply a materially later timestamp describing a different state of the same fact)? -> supersede. +3. Do two or more rows assert mutually exclusive facts with no textual or temporal signal indicating which one is current? -> contradict. +4. None of the above apply to any rows in this cluster? -> skip. +When it is genuinely ambiguous whether a pair of rows should be merged or superseded, prefer supersede: it is the safer, fully-reversible choice, since a superseded row is retained as historical record rather than combined away into a single new record. + +"events" and "cases" categories are append-only: they can never be superseded or contradicted (append-only means invalidation-protection, not merge-immunity). A merge must never mix an append-only row with a non-append-only row, or with a different append-only category. The one exception: near-identical duplicate rows within the SAME append-only category (for example two "events" rows describing the exact same occurrence, or two "cases" rows describing the exact same problem/solution) may still be merged like any other true duplicate. Outside that same-category duplicate case, leave append-only rows out of absorbed_indices, with one directional exception: an append-only row MAY serve as the supersede survivor_index when every absorbed row is non-append-only -- the append-only row itself is never written, only the stale mutable rows get marked no longer current. None of this ever blocks you from merging or superseding the OTHER, actionable rows in the same cluster. + +Rows in DIFFERENT non-append-only categories (profile, preferences, entities, patterns) are fully actionable against each other -- differing categories alone are never a reason to skip. Merge them when they state the same fact, choosing the more authoritative category's row as survivor (for identity facts like the user's name, profile over preferences); supersede when they conflict about the same fact, choosing the factually current row as survivor. Factual currency always decides supersede direction: never make a stale row the survivor for category reasons, and when the stale side is append-only (so it cannot be absorbed), use skip rather than a wrong-direction supersede. + +Source legend: legacy = pre-smart-format rows, manual = operator memory_store saves, auto-capture = extraction lane, reflection* = mirror lanes; manual rows are operator-authored and strong survivor candidates. + +Each member below also shows its timestamp (and valid_from when it differs) -- use these to judge supersede recency explicitly rather than inferring it from wording alone. + +Return JSON only: +{ + "verdicts": [ + { "cluster_index": 1, "verdict": "skip|merge|supersede|contradict", "survivor_index": 1, "absorbed_indices": [2, 3], "reason": "short explanation" } + ] +} + +Include exactly one verdict object per cluster listed below, each tagged with the matching cluster_index. Only include survivor_index and absorbed_indices for merge or supersede. survivor_index and every entry in absorbed_indices are row numbers scoped to that cluster's own member list. absorbed_indices must never contain an append-only (events/cases) row -- unless the verdict is merge and every row in survivor_index/absorbed_indices shares the exact same append-only category. An append-only row may appear as survivor_index only for that same-category duplicate merge, or for a supersede whose absorbed rows are all non-append-only.`; + + const user = clusters + .map( + (c) => + `Cluster ${c.clusterIndex} members:\n\n${c.members + .map((m) => `${formatMemberHeader(m)}\n${formatMemberTiers(m)}`) + .join("\n\n")}` + ) + .join("\n\n===\n\n"); + + return { system, user }; +} + +export interface ConsolidateBatchMergeJob { + category: string; + existing: { abstract: string; overview: string; content: string }; + /** Every absorbed member folding into this job's existing memory. */ + additions: Array<{ abstract: string; overview: string; content: string }>; +} + +/** + * Formats one labelled field for a numbered prompt block: the field on its + * own 3-space-indented line, multi-line values split per line with any + * leading markdown list-marker run (`- ` / `* `, repeated) stripped while + * the line's own inner indentation is kept, and every continuation line + * indented under the block. Other content markdown (e.g. `##` headings) is + * deliberately left as-is. + */ +function formatIndentedFieldLines(label: string, value: string): string[] { + const valueLines = String(value ?? "") + .split("\n") + .map((line) => line.replace(/^(\s*)(?:[-*] )+/, "$1")); + const lines = [` ${label}: ${valueLines[0]}`]; + for (const continuation of valueLines.slice(1)) { + lines.push(` ${continuation}`); + } + return lines; +} + +/** + * Batched variant of the consolidate merge writer prompt: one LLM call + * writes every numbered merge job. Each job carries its survivor ("Existing + * memory") and every absorbed member folding into it ("New information"); + * merge requirements match CONSOLIDATE_MERGE_SYSTEM_PROMPT verbatim — only + * the call topology changes from one call per absorbed member to one call + * per batch of merge verdicts. + */ +export function buildConsolidateBatchMergePrompt(jobs: ConsolidateBatchMergeJob[]): SplitPrompt { + const system = `You are a memory consolidation merge writer. Merge each numbered job below into a single coherent record with all three levels (abstract, overview, content). For each job, merge every "New information" section into that job's "Existing memory"; never mix content across jobs. + +Requirements: +- Remove duplicate information +- Keep the most up-to-date details +- Maintain a coherent narrative +- Keep code identifiers, URIs, and model names unchanged when they are proper nouns + +Return JSON only, with exactly one entry per job, in this shape: +{ + "results": [ + { "index": 1, "abstract": "Merged one-line abstract", "overview": "Merged structured Markdown overview", "content": "Merged full content" } + ] +} + +- "index" is the job's number in the batch below.`; + + const blocks = jobs.map((job, i) => { + const lines = [`${i + 1}. Category: ${job.category}`, ` Existing memory:`]; + lines.push(...formatIndentedFieldLines("Abstract", job.existing.abstract)); + lines.push(...formatIndentedFieldLines("Overview", job.existing.overview)); + lines.push(...formatIndentedFieldLines("Content", job.existing.content)); + job.additions.forEach((addition, j) => { + lines.push(job.additions.length > 1 ? ` New information ${j + 1}:` : ` New information:`); + lines.push(...formatIndentedFieldLines("Abstract", addition.abstract)); + lines.push(...formatIndentedFieldLines("Overview", addition.overview)); + lines.push(...formatIndentedFieldLines("Content", addition.content)); + }); + return lines.join("\n"); + }); + + const user = `Merge jobs: + +${blocks.join("\n\n")}`; + + return { system, user }; +} diff --git a/src/llm-client.ts b/src/llm-client.ts index 2e21a20ab..b538b869c 100644 --- a/src/llm-client.ts +++ b/src/llm-client.ts @@ -27,9 +27,20 @@ export interface LlmClientConfig { warnLog?: (msg: string) => void; } +const DEFAULT_SYSTEM_PROMPT = + "You are a memory extraction assistant. Always respond with valid JSON only."; + export interface LlmClient { - /** Send a prompt and parse the JSON response. Returns null on failure. */ - completeJson(prompt: string, label?: string): Promise; + /** + * Send a prompt and parse the JSON response. Returns null on failure. + * `systemPrompt`, when provided, replaces the default generic system + * message with a stage-specific identity/instructions block. `temperature`, + * when provided, overrides the client's default sampling temperature for + * this call only (e.g. 0 for callers that need reproducible output). The + * OAuth client's responses API has no temperature parameter, so it accepts + * and ignores this argument. + */ + completeJson(prompt: string, label?: string, systemPrompt?: string, temperature?: number): Promise; /** Best-effort diagnostics for the most recent failure, if any. */ getLastError(): string | null; } @@ -232,7 +243,7 @@ function createApiKeyClient(config: LlmClientConfig, log: (msg: string) => void, let lastError: string | null = null; return { - async completeJson(prompt: string, label = "generic"): Promise { + async completeJson(prompt: string, label = "generic", systemPrompt?: string, temperature?: number): Promise { lastError = null; try { const request = { @@ -240,12 +251,11 @@ function createApiKeyClient(config: LlmClientConfig, log: (msg: string) => void, messages: [ { role: "system", - content: - "You are a memory extraction assistant. Always respond with valid JSON only.", + content: systemPrompt ?? DEFAULT_SYSTEM_PROMPT, }, { role: "user", content: prompt }, ], - temperature: 0.1, + temperature: temperature ?? 0.1, ...(shouldDisableReasoningForJson(config.model) ? { chat_template_kwargs: { enable_thinking: false } } : {}), @@ -351,7 +361,7 @@ function createOauthClient(config: LlmClientConfig, log: (msg: string) => void, } return { - async completeJson(prompt: string, label = "generic"): Promise { + async completeJson(prompt: string, label = "generic", systemPrompt?: string, _temperature?: number): Promise { lastError = null; try { const session = await getSession(); @@ -371,8 +381,7 @@ function createOauthClient(config: LlmClientConfig, log: (msg: string) => void, signal, body: JSON.stringify({ model: normalizeOauthModel(config.model), - instructions: - "You are a memory extraction assistant. Always respond with valid JSON only.", + instructions: systemPrompt ?? DEFAULT_SYSTEM_PROMPT, input: [ { role: "user", diff --git a/src/store.ts b/src/store.ts index 4a6d93b94..48906b1d3 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1735,7 +1735,9 @@ export class MemoryStore { const safeLimit = clampInt(limit, 1, 20); // Over-fetch more aggressively when filtering inactive records, // because superseded historical rows can crowd out active ones. - const inactiveFilter = options?.excludeInactive ?? false; + // excludeInactive defaults to true: invalidated/superseded rows are + // invisible unless a caller opts out explicitly (item 6, PR #946). + const inactiveFilter = options?.excludeInactive ?? true; const overFetchMultiplier = inactiveFilter ? 20 : 10; const fetchLimit = Math.min(safeLimit * overFetchMultiplier, 200); @@ -1829,7 +1831,8 @@ export class MemoryStore { if (isExplicitDenyAllScopeFilter(scopeFilter)) return []; const safeLimit = clampInt(limit, 1, 20); - const inactiveFilter = options?.excludeInactive ?? false; + // excludeInactive defaults to true: see vectorSearch above (item 6, PR #946). + const inactiveFilter = options?.excludeInactive ?? true; // Over-fetch when filtering inactive records to avoid crowding const fetchLimit = inactiveFilter ? Math.min(safeLimit * 20, 200) : safeLimit; @@ -1949,8 +1952,9 @@ export class MemoryStore { const metadata = parseSmartMetadata(entry.metadata, entry); - // Skip inactive (superseded) records when requested - if (options?.excludeInactive && !isMemoryActiveAt(metadata)) { + // Skip inactive (superseded) records unless explicitly opted out + // (excludeInactive defaults to true -- item 6, PR #946). + if ((options?.excludeInactive ?? true) && !isMemoryActiveAt(metadata)) { continue; } @@ -2039,6 +2043,7 @@ export class MemoryStore { category?: string, limit = 20, offset = 0, + options?: { excludeInactive?: boolean }, ): Promise { await this.ensureInitialized(); @@ -2092,11 +2097,18 @@ export class MemoryStore { }), ); + // excludeInactive defaults to true: invalidated/superseded rows are + // invisible to list() unless a caller opts out explicitly (item 6, PR #946). + const excludeInactive = options?.excludeInactive ?? true; + const activeEntries = excludeInactive + ? entries.filter((entry) => isMemoryActiveAt(parseSmartMetadata(entry.metadata, entry))) + : entries; + return (category - ? entries.filter((entry) => + ? activeEntries.filter((entry) => matchesMemoryCategoryFilter(entry.category, category, entry.metadata), ) - : entries) + : activeEntries) .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)) .slice(offset, offset + limit); } @@ -2121,6 +2133,7 @@ export class MemoryStore { async stats(scopeFilter?: string[]): Promise<{ totalCount: number; + liveCount: number; scopeCounts: Record; categoryCounts: Record; }> { @@ -2130,6 +2143,7 @@ export class MemoryStore { if (isExplicitDenyAllScopeFilter(scopeFilter)) { return { totalCount: 0, + liveCount: 0, scopeCounts: {}, categoryCounts: {}, }; @@ -2146,13 +2160,18 @@ export class MemoryStore { const applyConditions = (query: any) => conditions.length > 0 ? query.where(conditions.join(" AND ")) : query; + // scopeCounts/categoryCounts stay blended (total, historical record + // included) -- only the top-level total/live split is added here, per + // item 6 (PR #946): "report a live vs total split rather than one + // blended count." const results = await this.queryRowsWithProjectionFallback( applyConditions, - ["scope", "category"], + ["scope", "category", "metadata", "timestamp"], ); const scopeCounts: Record = {}; const categoryCounts: Record = {}; + let liveCount = 0; for (const row of results) { const scope = (row.scope as string | undefined) ?? "global"; @@ -2160,10 +2179,16 @@ export class MemoryStore { scopeCounts[scope] = (scopeCounts[scope] || 0) + 1; categoryCounts[category] = (categoryCounts[category] || 0) + 1; + + const metadata = parseSmartMetadata((row.metadata as string) || "{}", { + timestamp: normalizeMemoryTimestamp(row.timestamp, 0), + }); + if (isMemoryActiveAt(metadata)) liveCount += 1; } return { totalCount: results.length, + liveCount, scopeCounts, categoryCounts, }; @@ -2658,6 +2683,7 @@ export class MemoryStore { maxTimestamp: number, scopeFilter?: string[], limit = 200, + options?: { excludeInactive?: boolean }, ): Promise { await this.ensureInitialized(); @@ -2677,7 +2703,7 @@ export class MemoryStore { .where(whereClause) .toArray(); - return results + const entries = results .map( (row): MemoryEntry => ({ id: row.id as string, @@ -2689,7 +2715,16 @@ export class MemoryStore { timestamp: normalizeMemoryTimestamp(row.timestamp, 0), metadata: (row.metadata as string) || "{}", }), - ) + ); + + // excludeInactive defaults to true: a background compactor or + // consolidate run must not cluster already-dead rows (item 6, PR #946). + const excludeInactive = options?.excludeInactive ?? true; + const activeEntries = excludeInactive + ? entries.filter((entry) => isMemoryActiveAt(parseSmartMetadata(entry.metadata, entry))) + : entries; + + return activeEntries .sort((a, b) => b.timestamp - a.timestamp) .slice(0, limit); } diff --git a/src/tools.ts b/src/tools.ts index acbe2be9a..9fb7f2218 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -2235,6 +2235,9 @@ export function registerMemoryListTool( description: "Number of memories to skip (default: 0)", }), ), + includeInvalidated: Type.Optional( + Type.Boolean({ description: "Include invalidated/superseded rows (default false)." }), + ), }), async execute(_toolCallId, params, _signal, _onUpdate, runtimeCtx) { const { @@ -2242,11 +2245,13 @@ export function registerMemoryListTool( scope, category, offset = 0, + includeInvalidated = false, } = params as { limit?: number; scope?: string; category?: string; offset?: number; + includeInvalidated?: boolean; }; try { @@ -2263,6 +2268,7 @@ export function registerMemoryListTool( category, safeLimit, safeOffset, + { excludeInactive: !includeInvalidated }, ); if (entries.length === 0) { @@ -2769,12 +2775,16 @@ export function registerMemoryCompactTool( scope: Type.Optional(Type.String({ description: "Optional scope filter." })), dryRun: Type.Optional(Type.Boolean({ description: "Preview compaction only (default true)." })), limit: Type.Optional(Type.Number({ description: "Max entries to scan (default 200)." })), + includeInvalidated: Type.Optional( + Type.Boolean({ description: "Include invalidated/superseded rows in the scan (default false)." }), + ), }), async execute(_toolCallId, params, _signal, _onUpdate, runtimeCtx) { - const { scope, dryRun = true, limit = 200 } = params as { + const { scope, dryRun = true, limit = 200, includeInvalidated = false } = params as { scope?: string; dryRun?: boolean; limit?: number; + includeInvalidated?: boolean; }; const safeLimit = clampInt(limit, 20, 1000); @@ -2790,7 +2800,13 @@ export function registerMemoryCompactTool( scopeFilter = [scope]; } - const entries = await runtimeContext.store.list(scopeFilter, undefined, safeLimit, 0); + const entries = await runtimeContext.store.list( + scopeFilter, + undefined, + safeLimit, + 0, + { excludeInactive: !includeInvalidated }, + ); const canonicalByKey = new Map(); const duplicates: Array<{ duplicateId: string; canonicalId: string; key: string }> = []; diff --git a/test/invalidated-rows-visibility.test.mjs b/test/invalidated-rows-visibility.test.mjs new file mode 100644 index 000000000..43f63a40a --- /dev/null +++ b/test/invalidated-rows-visibility.test.mjs @@ -0,0 +1,401 @@ +// test/invalidated-rows-visibility.test.mjs +// +// Item 6 (PR #946 fix round): caller-level coverage for the store-layer +// excludeInactive default (test/store-excludeinactive-default.test.mjs +// covers the store.ts choke point itself). This file covers the CLI and +// tools.ts consumers that need explicit opt-in/opt-out wiring on top of +// the new default: CLI export/list/obsidian, and the memory_list/ +// memory_debug/memory_compact tools. + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import jitiFactory from "jiti"; +import { Command } from "commander"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +function makeStore(prefix, vectorDim = 4) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + const { MemoryStore } = jiti("../src/store.ts"); + return { store: new MemoryStore({ dbPath: dir, vectorDim }), dir }; +} + +async function storeLiveAndInvalidatedPair(store) { + const live = await store.store({ + text: "Live fact: user likes cola", + vector: [1, 0, 0, 0], + category: "preference", + scope: "test", + importance: 0.7, + metadata: JSON.stringify({ + l0_abstract: "Live fact: user likes cola", + memory_category: "preferences", + valid_from: Date.now() - 10_000, + }), + }); + + const dead = await store.store({ + text: "Dead fact: user liked tea (superseded)", + vector: [1, 0, 0, 0], + category: "preference", + scope: "test", + importance: 0.7, + metadata: JSON.stringify({ + l0_abstract: "Dead fact: user liked tea (superseded)", + memory_category: "preferences", + valid_from: Date.now() - 20_000, + }), + }); + + await store.update(dead.id, { + metadata: JSON.stringify({ + l0_abstract: "Dead fact: user liked tea (superseded)", + memory_category: "preferences", + valid_from: Date.now() - 20_000, + invalidated_at: Date.now() - 5_000, + superseded_by: live.id, + }), + }); + + return { live, dead }; +} + +describe("item 6: CLI export/list/obsidian invalidated-row visibility", () => { + it("export includes invalidated rows by default (backup/export exception)", async () => { + const { store, dir } = makeStore("item6-export-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { createMemoryCLI } = jiti("../cli.ts"); + + const outFile = join(dir, "export.json"); + const context = { store, retriever: {}, scopeManager: {}, migrator: {} }; + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + await program.parseAsync([ + "node", "openclaw", "memory-pro", "export", "--scope", "test", "--output", outFile, + ]); + + const exported = JSON.parse(readFileSync(outFile, "utf8")); + const ids = exported.memories.map((m) => m.id); + assert.ok(ids.includes(live.id), "export must include the live row"); + assert.ok(ids.includes(dead.id), "export must include the invalidated row (backup semantics)"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("list excludes invalidated rows by default", async () => { + const { store, dir } = makeStore("item6-list-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { createMemoryCLI } = jiti("../cli.ts"); + + const context = { store, retriever: {}, scopeManager: {}, migrator: {} }; + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + // list --json writes via process.stdout.write (writeJson/writeStdout), + // not console.log, and pretty-prints (JSON.stringify(obj, null, 2)) -- + // capture the raw stdout chunks instead of console.log lines. + const chunks = []; + const originalWrite = process.stdout.write.bind(process.stdout); + process.stdout.write = (chunk) => { + chunks.push(String(chunk)); + return true; + }; + + try { + await program.parseAsync(["node", "openclaw", "memory-pro", "list", "--scope", "test", "--json"]); + } finally { + process.stdout.write = originalWrite; + } + + const listed = JSON.parse(chunks.join("")); + const ids = listed.map((m) => m.id); + assert.ok(ids.includes(live.id), "list must include the live row"); + assert.ok(!ids.includes(dead.id), "list must exclude the invalidated row by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("list --include-invalidated surfaces both rows", async () => { + const { store, dir } = makeStore("item6-list-optin-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { createMemoryCLI } = jiti("../cli.ts"); + + const context = { store, retriever: {}, scopeManager: {}, migrator: {} }; + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + const chunks = []; + const originalWrite = process.stdout.write.bind(process.stdout); + process.stdout.write = (chunk) => { + chunks.push(String(chunk)); + return true; + }; + + try { + await program.parseAsync([ + "node", "openclaw", "memory-pro", "list", "--scope", "test", "--json", "--include-invalidated", + ]); + } finally { + process.stdout.write = originalWrite; + } + + const listed = JSON.parse(chunks.join("")); + const ids = listed.map((m) => m.id); + assert.ok(ids.includes(live.id)); + assert.ok(ids.includes(dead.id), "--include-invalidated must surface the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("obsidian excludes invalidated rows by default and surfaces them with --include-invalidated", async () => { + const { store, dir } = makeStore("item6-obsidian-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { createMemoryCLI } = jiti("../cli.ts"); + const vaultDefault = join(dir, "vault-default"); + const vaultOptIn = join(dir, "vault-optin"); + + const context = { store, retriever: {}, scopeManager: {}, migrator: {} }; + + const programDefault = new Command(); + programDefault.exitOverride(); + createMemoryCLI(context)({ program: programDefault }); + await programDefault.parseAsync([ + "node", "openclaw", "memory-pro", "sync", "obsidian", "--vault", vaultDefault, "--scope", "test", + ]); + + const programOptIn = new Command(); + programOptIn.exitOverride(); + createMemoryCLI(context)({ program: programOptIn }); + await programOptIn.parseAsync([ + "node", "openclaw", "memory-pro", "sync", "obsidian", "--vault", vaultOptIn, "--scope", "test", "--include-invalidated", + ]); + + const fs = await import("node:fs"); + function listNoteBasenames(vaultPath) { + const root = join(vaultPath, "00-AI-Memory"); + const files = []; + for (const catDir of fs.readdirSync(root)) { + const catPath = join(root, catDir); + if (!fs.statSync(catPath).isDirectory()) continue; + for (const f of fs.readdirSync(catPath)) files.push(f); + } + return files; + } + + const defaultNotes = listNoteBasenames(vaultDefault); + const optInNotes = listNoteBasenames(vaultOptIn); + + const liveShortId = live.id.slice(0, 12); + const deadShortId = dead.id.slice(0, 12); + + assert.ok(defaultNotes.some((f) => f.includes(liveShortId)), "default vault must contain the live note"); + assert.ok(!defaultNotes.some((f) => f.includes(deadShortId)), "default vault must NOT contain the invalidated note"); + + assert.ok(optInNotes.some((f) => f.includes(liveShortId))); + assert.ok(optInNotes.some((f) => f.includes(deadShortId)), "--include-invalidated vault must contain the invalidated note"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // NOTE: CLI search already excludes invalidated rows by default (it + // routes through retriever.ts, which hardcodes { excludeInactive: true } + // on every vectorSearch/bm25Search call) -- covered below. A --include- + // invalidated opt-in for search specifically (mirroring list/obsidian) is + // PUNTED for this round: it would require threading a new option through + // retriever.ts's private vectorOnlyRetrieval/hybridRetrieval/ + // bm25OnlyRetrieval helper chain, which is materially higher-risk (the + // primary recall/prompt-injection path) for a forensic-only nice-to-have + // that isn't in item 6's required acceptance list. + it("CLI search excludes invalidated rows by default (already correct; no code change needed)", async () => { + const { store, dir } = makeStore("item6-search-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { createRetriever } = jiti("../src/retriever.ts"); + const { createMemoryCLI } = jiti("../cli.ts"); + + const fakeEmbedder = { + embedQuery: async () => [1, 0, 0, 0], + embedPassage: async () => [1, 0, 0, 0], + }; + const retriever = createRetriever(store, fakeEmbedder, { minScore: 0 }); + + const context = { store, retriever, scopeManager: {}, migrator: {}, embedder: fakeEmbedder }; + + const chunks = []; + const originalWrite = process.stdout.write.bind(process.stdout); + process.stdout.write = (chunk) => { + chunks.push(String(chunk)); + return true; + }; + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + try { + await program.parseAsync([ + "node", "openclaw", "memory-pro", "search", "cola", "--scope", "test", "--json", + ]); + } finally { + process.stdout.write = originalWrite; + } + const results = JSON.parse(chunks.join("")); + const ids = results.map((r) => r.entry?.id ?? r.id); + assert.ok(ids.includes(live.id), "search must include the live row by default"); + assert.ok(!ids.includes(dead.id), "search must exclude the invalidated row by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("item 6: memory_list / memory_compact tool visibility", () => { + function scopeManagerFor(scope) { + return { + getAccessibleScopes: () => [scope], + getScopeFilter: () => [scope], + isAccessible: (s) => s === scope, + getDefaultScope: () => scope, + }; + } + + function toolFactory(store, scope) { + const { registerAllMemoryTools } = jiti("../src/tools.ts"); + const creators = new Map(); + const api = { + registerTool(factory, meta) { + creators.set(meta.name, factory); + }, + logger: { info() {}, warn() {}, debug() {} }, + }; + const context = { + agentId: "main", + store, + scopeManager: scopeManagerFor(scope), + retriever: {}, + embedder: { async embedPassage() { return [1, 0, 0, 0]; } }, + }; + registerAllMemoryTools(api, context, { enableManagementTools: true }); + return { + get(name) { + const factory = creators.get(name); + assert.ok(factory, `tool ${name} should be registered`); + return factory({}); + }, + }; + } + + it("memory_list excludes invalidated rows by default and surfaces them with includeInvalidated", async () => { + const { store, dir } = makeStore("item6-tool-list-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const tools = toolFactory(store, "test"); + const memoryList = tools.get("memory_list"); + + const defaultResult = await memoryList.execute("call-1", { scope: "test", limit: 50 }, undefined, undefined, {}); + const defaultIds = defaultResult.details.memories.map((m) => m.id); + assert.ok(defaultIds.includes(live.id), "memory_list must include the live row by default"); + assert.ok(!defaultIds.includes(dead.id), "memory_list must exclude the invalidated row by default"); + + const optInResult = await memoryList.execute( + "call-2", + { scope: "test", limit: 50, includeInvalidated: true }, + undefined, + undefined, + {}, + ); + const optInIds = optInResult.details.memories.map((m) => m.id); + assert.ok(optInIds.includes(live.id)); + assert.ok(optInIds.includes(dead.id), "includeInvalidated:true must surface the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("memory_compact scans only live rows by default and includes invalidated rows with includeInvalidated", async () => { + const { store, dir } = makeStore("item6-tool-compact-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const tools = toolFactory(store, "test"); + const memoryCompact = tools.get("memory_compact"); + + const defaultResult = await memoryCompact.execute( + "call-1", + { scope: "test", dryRun: true, limit: 200 }, + undefined, + undefined, + {}, + ); + assert.equal(defaultResult.details.scanned, 1, "memory_compact must scan only the live row by default"); + + const optInResult = await memoryCompact.execute( + "call-2", + { scope: "test", dryRun: true, limit: 200, includeInvalidated: true }, + undefined, + undefined, + {}, + ); + assert.equal(optInResult.details.scanned, 2, "includeInvalidated:true must scan both rows"); + void live; + void dead; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("item 6: admission-control novelty gate scores against live rows only", () => { + it("loadRelevantMatches (via evaluate()) never compares the candidate against an invalidated row", async () => { + const { store, dir } = makeStore("item6-admission-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const { AdmissionController, DEFAULT_ADMISSION_CONTROL_CONFIG } = jiti("../src/admission-control.ts"); + + const fakeLlm = { + async completeJson() { + return { utility: 0.5, reason: "test stub" }; + }, + getLastError() { + return null; + }, + }; + + const controller = new AdmissionController(store, fakeLlm, DEFAULT_ADMISSION_CONTROL_CONFIG); + + const evaluation = await controller.evaluate({ + candidate: { + category: "preferences", + abstract: "User likes cola", + overview: "", + content: "User likes cola", + }, + candidateVector: [1, 0, 0, 0], + conversationText: "User: I like cola.", + scopeFilter: ["test"], + }); + + const comparedIds = evaluation.audit.compared_existing_memory_ids || []; + assert.ok(comparedIds.includes(live.id), "novelty scoring must compare against the live row"); + assert.ok( + !comparedIds.includes(dead.id), + "novelty scoring must NOT compare against the invalidated row (item 6)", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/memory-consolidate-admission-independence.test.mjs b/test/memory-consolidate-admission-independence.test.mjs new file mode 100644 index 000000000..0ce2c99e1 --- /dev/null +++ b/test/memory-consolidate-admission-independence.test.mjs @@ -0,0 +1,103 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; +import { Command } from "commander"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +let nextId = 1; +function makeRow({ scope = "global", abstract, content, factKey, vector, timestamp = 1_700_000_000_000 }) { + const id = `row-${String(nextId++).padStart(6, "0")}`; + const metadata = { + l0_abstract: abstract, + l1_overview: "", + l2_content: content || abstract, + memory_category: "preferences", + fact_key: factKey, + source: "manual", + valid_from: timestamp, + }; + return { id, text: abstract, vector, category: "preference", scope, importance: 0.7, timestamp, metadata: JSON.stringify(metadata) }; +} + +// Every method throws if ever invoked -- a "poison pill" proving consolidate +// never reaches for admission control at any point in its flow. +function makePoisonAdmissionController() { + const poison = (name) => () => { + throw new Error(`consolidate must never touch AdmissionController.${name}`); + }; + return { + evaluate: poison("evaluate"), + evaluateBatch: poison("evaluateBatch"), + getAdmissionController: poison("getAdmissionController"), + }; +} + +describe("memory consolidate: item 9 admissionControl independence", () => { + it("runs the full consolidate flow identically with admissionControl.enabled:false and a poison-pill controller never invoked", async () => { + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ scope: "agent:testbot", abstract: "Coffee order: oat milk latte", content: "a", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ scope: "agent:testbot", abstract: "Coffee order: oat milk latte, extra hot", content: "b", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; + + const calls = []; + const poisonAdmissionController = makePoisonAdmissionController(); + const context = { + store: { + fetchForCompaction: async (maxTimestamp, scopeFilter, limit) => + rows.filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp).slice(0, limit ?? rows.length), + update: async (id, patch) => { + const row = rows.find((r) => r.id === id); + if (row) Object.assign(row, patch); + return row ? { ...row } : null; + }, + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + }, + retriever: {}, + scopeManager: {}, + migrator: {}, + embedder: { embedPassage: async () => [1, 0] }, + llmClient: { + completeJson: async (_prompt, label) => { + calls.push(label); + if (label === "consolidate-decide") { + return { verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same fact, second adds detail" }] }; + } + return { + results: [ + { index: 1, abstract: "Coffee order: oat milk latte, extra hot", overview: "", content: "merged content" }, + ], + }; + }, + getLastError: () => null, + }, + // Not declared on CLIContext's TS shape -- present at runtime the way an + // externally-constructed controller would be, to prove that IF consolidate's + // action ever reached for it (directly or via some future refactor), this + // test would catch it immediately via the poison pill throwing. + admissionController: poisonAdmissionController, + pluginConfig: { admissionControl: { enabled: false } }, + }; + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + await program.parseAsync(["node", "openclaw", "memory-pro", "consolidate", "--agent", "testbot", "--apply", "--yes"]); + + assert.ok(calls.includes("consolidate-decide"), "the decider call must still fire normally"); + assert.ok(calls.includes("consolidate-merge-batch"), "merge-content generation must still fire normally"); + + const survivor = rows.find((r) => r.text === "Coffee order: oat milk latte, extra hot"); + assert.ok(survivor, "the merge must have actually applied, proving the flow completed end to end"); + }); +}); diff --git a/test/memory-consolidate-cost-gate.test.mjs b/test/memory-consolidate-cost-gate.test.mjs new file mode 100644 index 000000000..a4528a81e --- /dev/null +++ b/test/memory-consolidate-cost-gate.test.mjs @@ -0,0 +1,252 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; +import { EventEmitter } from "node:events"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +const { + runConsolidate, + computeConsolidateCostPreview, + formatConsolidateCostPreview, +} = jiti(path.join(testDir, "..", "src", "consolidate.ts")); + +const { createConsolidateConfirm } = jiti(path.join(testDir, "..", "cli.ts")); + +let nextId = 1; +function makeRow({ scope = "global", abstract, content, factKey, vector, timestamp = 1_700_000_000_000 }) { + const id = `row-${String(nextId++).padStart(6, "0")}`; + const metadata = { + l0_abstract: abstract, + l1_overview: "", + l2_content: content || abstract, + memory_category: "preferences", + fact_key: factKey, + source: "manual", + valid_from: timestamp, + }; + return { id, text: abstract, vector, category: "preference", scope, importance: 0.7, timestamp, metadata: JSON.stringify(metadata) }; +} + +function makeFakeStore(initialRows) { + const rows = initialRows.map((r) => ({ ...r })); + return { + rows, + fetchRows: async (scopeFilter, maxTimestamp, limit) => + rows.filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp).slice(0, limit).map((r) => ({ ...r })), + update: async (id, patch) => { + const row = rows.find((r) => r.id === id); + if (!row) return null; + if (patch.text !== undefined) row.text = patch.text; + if (patch.vector !== undefined) row.vector = patch.vector; + if (patch.metadata !== undefined) row.metadata = patch.metadata; + return { ...row }; + }, + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + embed: async (text) => [text.length, 0, 0], + }; +} + +function buildMergeableRows() { + const ts = 1_700_000_000_000; + return [ + makeRow({ abstract: "Coffee order: oat milk latte", content: "a", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", content: "b", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; +} + +describe("memory consolidate: cost preview (pure)", () => { + it("reports N clusters -> 1 batched decider call plus the chunk-capped batched merge-writer call count", () => { + // 3 units, each at most one merge job -> 3 jobs -> ceil(3/10) = 1 batched call + const units = [ + { members: [{}, {}] }, + { members: [{}, {}, {}] }, + { members: [{}, {}, {}, {}] }, + ]; + const preview = computeConsolidateCostPreview(units); + assert.equal(preview.clusterCount, 3); + assert.equal(preview.maxMergeJobs, 3); + assert.equal(preview.maxMergeContentCalls, 1); + }); + + it("chunk math: more units than the batch cap means more than one batched call", () => { + const units = Array.from({ length: 12 }, () => ({ members: [{}, {}] })); + const preview = computeConsolidateCostPreview(units); + assert.equal(preview.maxMergeJobs, 12); + assert.equal(preview.maxMergeContentCalls, 2); + }); + + it("formats the preview as a single line with real plural counts and one worst-case qualifier", () => { + const preview = { clusterCount: 4, maxMergeJobs: 4, maxMergeContentCalls: 1 }; + const text = formatConsolidateCostPreview(preview); + assert.equal( + text, + "4 clusters -> 1 batched decider call + worst case 1 batched merge-content call covering 4 merge jobs", + ); + assert.ok(!text.includes("\n"), "preview must be a single line"); + assert.doesNotMatch(text, /up to|\(s\)/); + }); + + it("uses singular nouns when every count is 1", () => { + const preview = { clusterCount: 1, maxMergeJobs: 1, maxMergeContentCalls: 1 }; + const text = formatConsolidateCostPreview(preview); + assert.equal( + text, + "1 cluster -> 1 batched decider call + worst case 1 batched merge-content call covering 1 merge job", + ); + }); + + it("omits the merge-writer line when no cluster could ever produce a merge", () => { + const preview = { clusterCount: 2, maxMergeJobs: 0, maxMergeContentCalls: 0 }; + const text = formatConsolidateCostPreview(preview); + assert.match(text, /2 cluster/); + assert.doesNotMatch(text, /merge-content/); + }); +}); + +describe("memory consolidate: item 7 cost gate (runConsolidate)", () => { + it("aborts before any LLM call when confirmCost declines, and never invokes completeJson", async () => { + const store = makeFakeStore(buildMergeableRows()); + let completeJsonCalls = 0; + const completeJson = async () => { + completeJsonCalls += 1; + return { verdicts: [] }; + }; + let confirmCostCalledWith = null; + + const result = await runConsolidate( + { + ...store, + completeJson, + confirmCost: async (message) => { + confirmCostCalledWith = message; + return false; + }, + }, + { scope: "global", apply: true, now: 1_700_100_000_000 }, + ); + + assert.equal(completeJsonCalls, 0, "no LLM call may fire when the cost gate is declined"); + assert.equal(result.status, "aborted", "the result must clearly signal the run never proceeded"); + assert.ok(confirmCostCalledWith, "confirmCost must have been called with a preview message"); + assert.match(confirmCostCalledWith, /1 cluster/); + assert.equal(store.rows.length, 2, "declining the gate must not touch the store"); + }); + + it("gate also covers dry-runs: apply:false still aborts on decline with zero LLM calls", async () => { + const store = makeFakeStore(buildMergeableRows()); + let completeJsonCalls = 0; + const completeJson = async () => { + completeJsonCalls += 1; + return { verdicts: [] }; + }; + + const result = await runConsolidate( + { ...store, completeJson, confirmCost: async () => false }, + { scope: "global", apply: false, now: 1_700_100_000_000 }, + ); + + assert.equal(completeJsonCalls, 0); + assert.equal(result.status, "aborted"); + }); + + it("--yes (autoConfirm) bypasses the gate entirely and never calls confirmCost", async () => { + const store = makeFakeStore(buildMergeableRows()); + let confirmCostCalls = 0; + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { verdicts: [{ cluster_index: 1, verdict: "skip", reason: "not a dup after all" }] }; + } + return null; + }; + + const result = await runConsolidate( + { + ...store, + completeJson, + confirmCost: async () => { + confirmCostCalls += 1; + return true; + }, + }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(confirmCostCalls, 0, "--yes must skip calling confirmCost at all"); + assert.equal(result.status, "completed"); + }); + + it("proceeds normally when confirmCost affirms", async () => { + const store = makeFakeStore(buildMergeableRows()); + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { verdicts: [{ cluster_index: 1, verdict: "skip", reason: "fine as-is" }] }; + } + return null; + }; + + const result = await runConsolidate( + { ...store, completeJson, confirmCost: async () => true }, + { scope: "global", apply: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.status, "completed"); + assert.equal(result.clusters.length, 1); + }); + + it("skips the gate entirely (no confirmCost call, no abort) when there are zero clusters to act on", async () => { + const store = makeFakeStore([makeRow({ abstract: "Solo fact", vector: [9, 9], timestamp: 1_700_000_000_000 })]); + let confirmCostCalls = 0; + const result = await runConsolidate( + { ...store, completeJson: async () => { throw new Error("must not be called"); }, confirmCost: async () => { confirmCostCalls += 1; return true; } }, + { scope: "global", apply: true, now: 1_700_100_000_000 }, + ); + assert.equal(confirmCostCalls, 0, "nothing to confirm when there are no clusters"); + assert.equal(result.status, "completed"); + assert.equal(result.clusters.length, 0); + }); + + it("treats a missing confirmCost dep as a safe decline (fail-safe default), not a crash", async () => { + const store = makeFakeStore(buildMergeableRows()); + let completeJsonCalls = 0; + const result = await runConsolidate( + { ...store, completeJson: async () => { completeJsonCalls += 1; return { verdicts: [] }; } }, + { scope: "global", apply: true, now: 1_700_100_000_000 }, + ); + assert.equal(completeJsonCalls, 0); + assert.equal(result.status, "aborted"); + }); +}); + +describe("memory consolidate: item 7 CLI default confirm (TTY detection)", () => { + function makeStream({ isTTY }) { + const stream = new EventEmitter(); + stream.isTTY = isTTY; + stream.write = () => true; + return stream; + } + + it("resolves false without reading anything when stdin is not a TTY", async () => { + const stdin = makeStream({ isTTY: false }); + const stdout = makeStream({ isTTY: true }); + const confirm = createConsolidateConfirm({ stdin, stdout }); + + const result = await confirm("Proceed?"); + assert.equal(result, false); + }); + + it("resolves false without reading anything when stdout is not a TTY", async () => { + const stdin = makeStream({ isTTY: true }); + const stdout = makeStream({ isTTY: false }); + const confirm = createConsolidateConfirm({ stdin, stdout }); + + const result = await confirm("Proceed?"); + assert.equal(result, false); + }); +}); diff --git a/test/memory-consolidate-polish.test.mjs b/test/memory-consolidate-polish.test.mjs new file mode 100644 index 000000000..35e4edb33 --- /dev/null +++ b/test/memory-consolidate-polish.test.mjs @@ -0,0 +1,342 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +const { runConsolidate, computeClusterFingerprint, formatConsolidatePlanForDisplay } = jiti( + path.join(testDir, "..", "src", "consolidate.ts"), +); +const { buildConsolidateBatchPrompt } = jiti( + path.join(testDir, "..", "src", "extraction-prompts.ts"), +); + +let nextId = 1; +function makeRow({ + scope = "global", + abstract, + content, + factKey, + vector, + category = "preferences", + timestamp = 1_700_000_000_000, +}) { + const id = `row-${String(nextId++).padStart(6, "0")}`; + const metadata = { + l0_abstract: abstract, + l1_overview: "", + l2_content: content || abstract, + memory_category: category, + fact_key: factKey, + source: "manual", + valid_from: timestamp, + }; + return { + id, + text: abstract, + vector, + category: "preference", + scope, + importance: 0.7, + timestamp, + metadata: JSON.stringify(metadata), + }; +} + +function makeFakeStore(initialRows) { + const rows = initialRows.map((r) => ({ ...r })); + return { + rows, + fetchRows: async (scopeFilter, maxTimestamp, limit) => + rows + .filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp) + .slice(0, limit) + .map((r) => ({ ...r })), + update: async (id, patch) => { + const row = rows.find((r) => r.id === id); + if (!row) return null; + if (patch.text !== undefined) row.text = patch.text; + if (patch.vector !== undefined) row.vector = patch.vector; + if (patch.metadata !== undefined) row.metadata = patch.metadata; + return { ...row }; + }, + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + embed: async (text) => [text.length, 0, 0], + }; +} + +function skipPairRows() { + const ts = 1_700_000_000_000; + return [ + makeRow({ abstract: "Tea order: green tea", content: "a", factKey: "preferences:tea order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Tea order: green tea please", content: "b", factKey: "preferences:tea order", vector: [1, 0], timestamp: ts + 1000 }), + ]; +} + +describe("consolidate polish: cluster fingerprints", () => { + it("is stable across member order and changes when any member's metadata changes", () => { + const a = { id: "row-1", metadata: "meta-a" }; + const b = { id: "row-2", metadata: "meta-b" }; + const fp1 = computeClusterFingerprint([a, b]); + const fp2 = computeClusterFingerprint([b, a]); + assert.equal(fp1, fp2, "member order must not affect the fingerprint"); + const fp3 = computeClusterFingerprint([a, { id: "row-2", metadata: "meta-b-changed" }]); + assert.notEqual(fp1, fp3, "a metadata change must change the fingerprint"); + const fp4 = computeClusterFingerprint([a]); + assert.notEqual(fp1, fp4, "a different member set must change the fingerprint"); + }); +}); + +describe("consolidate polish: convergence to zero via settled fingerprints", () => { + it("reports skip verdicts as newly settled, and a rerun with those fingerprints spends zero LLM calls and reports zero clusters", async () => { + const rows = skipPairRows(); + let llmCalls = 0; + const llm = async (_prompt, label) => { + llmCalls += 1; + if (label === "consolidate-decide") { + return { verdicts: [{ cluster_index: 1, verdict: "skip", reason: "both rows already agree" }] }; + } + return { results: [] }; + }; + + const run1 = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + assert.equal(run1.clusters.length, 1); + assert.equal(run1.newlySettled.length, 1, "a decided skip cluster must be reported as newly settled"); + assert.ok(llmCalls > 0); + + const callsAfterRun1 = llmCalls; + const run2 = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm }, + { + scope: "global", + apply: false, + autoConfirm: true, + now: 1_700_100_000_000, + settledFingerprints: new Set(run1.newlySettled), + }, + ); + assert.equal(llmCalls, callsAfterRun1, "settled clusters must not spend any LLM call"); + assert.equal(run2.clusters.length, 0, "settled clusters must not reappear as candidates"); + assert.equal(run2.settledSkipped, 1, "the settled cluster must be counted"); + assert.equal(run2.newlySettled.length, 0); + }); + + it("re-opens a settled cluster when a member row's metadata changes", async () => { + const rows = skipPairRows(); + const llm = async (_prompt, label) => + label === "consolidate-decide" + ? { verdicts: [{ cluster_index: 1, verdict: "skip", reason: "agree" }] } + : { results: [] }; + + const run1 = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + const mutated = rows.map((r, i) => + i === 0 ? { ...r, metadata: r.metadata.replace("green tea", "black tea") } : r, + ); + const run2 = await runConsolidate( + { ...makeFakeStore(mutated), completeJson: llm }, + { + scope: "global", + apply: false, + autoConfirm: true, + now: 1_700_100_000_000, + settledFingerprints: new Set(run1.newlySettled), + }, + ); + assert.equal(run2.settledSkipped, 0, "a changed member must re-open the cluster"); + assert.equal(run2.clusters.length, 1); + }); +}); + +describe("consolidate polish: append-only shield visibility", () => { + it("marks a shield-blocked verdict as blocked in the plan report and reports it as settled", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Deploy failed with ENOENT", content: "a", factKey: "cases:deploy failure", category: "cases", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "User prefers quick deploys", content: "b", factKey: "preferences:deploys", category: "preferences", vector: [1, 0], timestamp: ts + 1000 }), + ]; + const llm = async (_prompt, label) => + label === "consolidate-decide" + ? { verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same topic" }] } + : { results: [] }; + + const result = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.clusters.length, 1); + const cluster = result.clusters[0]; + assert.equal(cluster.action, null); + assert.equal(cluster.blocked, "append-only-shield", "shield-blocked verdicts must be marked, not silently actionless"); + assert.equal(result.newlySettled.length, 1, "a shield-blocked cluster is settled: rerunning cannot change the outcome"); + }); + + it("allows a supersede whose survivor is append-only, invalidates only the mutable absorbed row, and never writes the survivor", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Gym schedule changed to Tuesday and Friday evenings", content: "a", factKey: "events:gym schedule", category: "events", vector: [1, 0], timestamp: ts + 1000 }), + makeRow({ abstract: "Gym sessions happen Tuesday and Friday mornings", content: "b", factKey: "preferences:gym schedule", category: "preferences", vector: [1, 0], timestamp: ts }), + ]; + const store = makeFakeStore(rows); + const updatedIds = []; + const llm = async (_prompt, label) => + label === "consolidate-decide" + ? { verdicts: [{ cluster_index: 1, verdict: "supersede", survivor_index: 1, absorbed_indices: [2], reason: "evenings replaces mornings" }] } + : { results: [] }; + + const result = await runConsolidate( + { + ...store, + update: async (id, patch, scopeFilter) => { + updatedIds.push(id); + return store.update(id, patch, scopeFilter); + }, + completeJson: llm, + }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.applied.length, 1, "the direction-safe supersede must apply"); + assert.equal(result.clusters[0].blocked, undefined); + assert.deepEqual(updatedIds, [rows[1].id], "only the mutable absorbed row may be written"); + const absorbedMeta = JSON.parse(store.rows.find((r) => r.id === rows[1].id).metadata); + assert.equal(absorbedMeta.superseded_by, rows[0].id); + assert.ok(absorbedMeta.invalidated_at, "absorbed mutable row must be marked no longer current"); + assert.equal(store.rows.find((r) => r.id === rows[0].id).metadata, rows[0].metadata, "append-only survivor must stay byte-identical"); + }); + + it("still blocks a supersede that would absorb an append-only row", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Prefers evening gym sessions", content: "a", factKey: "preferences:gym schedule", category: "preferences", vector: [1, 0], timestamp: ts + 1000 }), + makeRow({ abstract: "Gym sessions moved to mornings", content: "b", factKey: "cases:gym schedule", category: "cases", vector: [1, 0], timestamp: ts }), + ]; + const llm = async (_prompt, label) => + label === "consolidate-decide" + ? { verdicts: [{ cluster_index: 1, verdict: "supersede", survivor_index: 1, absorbed_indices: [2], reason: "newer wins" }] } + : { results: [] }; + + const result = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.applied.length, 0); + assert.equal(result.clusters[0].blocked, "append-only-shield", "invalidating an append-only row stays forbidden"); + assert.equal(result.newlySettled.length, 1); + }); + + it("teaches the decider both the cross-category clause and the directional survivor exception", () => { + const { system } = buildConsolidateBatchPrompt([ + { + clusterIndex: 1, + members: [ + { index: 1, category: "profile", abstract: "User name: Sam Rivera", overview: "", content: "User name: Sam Rivera" }, + { index: 2, category: "preferences", abstract: "User's name is Sam Rivera.", overview: "", content: "User's name is Sam Rivera." }, + ], + }, + ]); + assert.match(system, /fully actionable against each other/); + assert.match(system, /differing categories alone are never a reason to skip/); + assert.match(system, /supersede whose absorbed rows are all non-append-only/); + assert.match(system, /never make a stale row the survivor for category reasons/i); + }); +}); + +describe("consolidate polish: honest failure classing", () => { + it("classes a null decide response as call-failed with one aggregate log line, not per-cluster malformed spam", async () => { + const rows = skipPairRows(); + const logs = []; + const llm = async (_prompt, label) => (label === "consolidate-decide" ? null : { results: [] }); + + const result = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm, log: (m) => logs.push(m) }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.clusters.length, 1); + assert.equal(result.clusters[0].failure, "call-failed"); + assert.equal(result.undecidedCallFailed, 1); + assert.equal(result.skippedMalformed, 0, "a failed call is not a malformed verdict"); + assert.equal(result.newlySettled.length, 0, "an undecided cluster must not settle"); + const callFailedLines = logs.filter((l) => l.includes("no response")); + assert.equal(callFailedLines.length, 1, "exactly one aggregate line for the failed call"); + assert.equal( + logs.filter((l) => l.includes("missing or malformed")).length, + 0, + "no per-cluster malformed spam when the whole call failed", + ); + }); + + it("still classes a genuinely missing verdict as malformed when the call itself succeeded", async () => { + const rows = skipPairRows(); + const logs = []; + const llm = async (_prompt, label) => + label === "consolidate-decide" ? { verdicts: [] } : { results: [] }; + + const result = await runConsolidate( + { ...makeFakeStore(rows), completeJson: llm, log: (m) => logs.push(m) }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.clusters[0].failure, "malformed-verdict"); + assert.equal(result.skippedMalformed, 1); + assert.equal(result.undecidedCallFailed, 0); + assert.equal(result.newlySettled.length, 0, "a malformed cluster must not settle"); + assert.ok(logs.some((l) => l.includes("missing or malformed"))); + }); +}); + +describe("consolidate polish: plan display shows every verdict class", () => { + const clusters = [ + { + clusterIndex: 1, + action: "merge", + memberIds: ["id-a", "id-b"], + memberTexts: ["Coffee: oat milk", "Coffee: oat milk latte"], + survivorId: "id-a", + absorbedIds: ["id-b"], + verdict: { verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same coffee fact" }, + mergedContent: { abstract: "Coffee: oat milk latte", overview: "o", content: "c" }, + }, + { + clusterIndex: 2, + blocked: "append-only-shield", + memberIds: ["id-c", "id-d"], + memberTexts: ["event one", "event two"], + verdict: { verdict: "supersede", survivor_index: 1, absorbed_indices: [2], reason: "newer event wins" }, + }, + { + clusterIndex: 3, + memberIds: ["id-e", "id-f"], + memberTexts: ["tea: green", "tea: green with honey"], + verdict: { verdict: "skip", reason: "distinct preparations" }, + }, + ]; + + it("headlines all three counts and lists skip verdicts with reason and member texts", () => { + const text = formatConsolidatePlanForDisplay(clusters); + assert.match(text, /Plan: 1 actionable cluster, 1 blocked, 1 skip\b/); + assert.match(text, /\[skip\] cluster 3 — distinct preparations/); + assert.match(text, /"tea: green with honey"/); + assert.match(text, /BLOCKED by append-only shield/); + assert.match(text, /"event two"/); + }); + + it("keeps the no-plan message when nothing was decided at all", () => { + assert.equal(formatConsolidatePlanForDisplay([]), "No actionable clusters in this plan."); + }); +}); diff --git a/test/memory-consolidate-two-phase-apply.test.mjs b/test/memory-consolidate-two-phase-apply.test.mjs new file mode 100644 index 000000000..fbe74f008 --- /dev/null +++ b/test/memory-consolidate-two-phase-apply.test.mjs @@ -0,0 +1,483 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +const { runConsolidate } = jiti(path.join(testDir, "..", "src", "consolidate.ts")); + +let nextId = 1; +function makeRow({ scope = "global", abstract, content, factKey, vector, timestamp = 1_700_000_000_000 }) { + const id = `row-${String(nextId++).padStart(6, "0")}`; + const metadata = { + l0_abstract: abstract, + l1_overview: "", + l2_content: content || abstract, + memory_category: "preferences", + fact_key: factKey, + source: "manual", + valid_from: timestamp, + }; + return { id, text: abstract, vector, category: "preference", scope, importance: 0.7, timestamp, metadata: JSON.stringify(metadata) }; +} + +function makeFakeStore(initialRows) { + const rows = initialRows.map((r) => ({ ...r })); + return { + rows, + fetchRows: async (scopeFilter, maxTimestamp, limit) => + rows.filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp).slice(0, limit).map((r) => ({ ...r })), + update: async (id, patch) => { + const row = rows.find((r) => r.id === id); + if (!row) return null; + if (patch.text !== undefined) row.text = patch.text; + if (patch.vector !== undefined) row.vector = patch.vector; + if (patch.metadata !== undefined) row.metadata = patch.metadata; + return { ...row }; + }, + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + embed: async (text) => [text.length, 0, 0], + }; +} + +function twoMemberMergeRows() { + const ts = 1_700_000_000_000; + return [ + makeRow({ abstract: "Coffee order: oat milk latte", content: "a", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", content: "b", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; +} + +function mergeDeciderLlm() { + let completeJsonCalls = 0; + const calls = []; + const completeJson = async (_prompt, label) => { + completeJsonCalls += 1; + calls.push(label); + if (label === "consolidate-decide") { + return { verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same fact, second row adds detail" }] }; + } + return { + results: [ + { index: 1, abstract: "Coffee order: oat milk latte, extra hot", overview: "", content: "merged content" }, + ], + }; + }; + return { completeJson, calls, callCount: () => completeJsonCalls }; +} + +describe("memory consolidate: item 8 plan building (merge content precomputed at plan time)", () => { + it("dry-run (apply:false) generates merge content during plan build, not just verdicts", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + + const result = await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.ok(llm.calls.includes("consolidate-merge-batch"), "merge content must be generated at plan-build time, even in dry-run"); + assert.equal(result.clusters[0].mergedContent?.abstract, "Coffee order: oat milk latte, extra hot"); + assert.equal(result.clusters[0].action, "merge"); + }); +}); + +describe("memory consolidate: item 8 two-phase apply (dry-run -> present -> confirm -> execute)", () => { + it("declining the apply prompt (anything other than true) makes zero store writes", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + let confirmApplyCalledWith = null; + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + confirmApply: async (message, clusters) => { + confirmApplyCalledWith = { message, clusters }; + return false; + }, + }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.ok(confirmApplyCalledWith, "confirmApply must be called with the full plan"); + assert.equal(confirmApplyCalledWith.clusters.length, 1); + assert.equal(confirmApplyCalledWith.clusters[0].action, "merge"); + assert.equal(confirmApplyCalledWith.clusters[0].survivorId, store.rows[0].id); + assert.deepEqual(confirmApplyCalledWith.clusters[0].absorbedIds, [store.rows[1].id]); + assert.equal(confirmApplyCalledWith.clusters[0].mergedContent.content, "merged content"); + + assert.equal(result.executed, false); + assert.equal(result.applied.length, 0); + assert.equal(store.rows[1].text, "Coffee order: oat milk latte, extra hot", "unmutated original text, not the merged text"); + assert.equal(JSON.parse(store.rows[1].metadata).invalidated_at, undefined, "no row may be invalidated when the user declines"); + }); + + it("confirming YES executes the plan as pure store operations, with the LLM dep provably not called again during execution", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + confirmApply: async () => true, + }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + const callsAfterBuild = llm.callCount(); + assert.equal(callsAfterBuild, 2, "exactly one decide call + one merge-content call during plan build"); + + assert.equal(result.executed, true); + assert.equal(result.applied.length, 1); + assert.equal(llm.callCount(), callsAfterBuild, "execution must call zero further LLM completions"); + + const survivor = store.rows.find((r) => r.id === result.applied[0].survivorId); + assert.equal(survivor.text, "Coffee order: oat milk latte, extra hot"); + const absorbed = store.rows.find((r) => r.id === result.applied[0].absorbedIds[0]); + assert.ok(JSON.parse(absorbed.metadata).invalidated_at, "absorbed row must be invalidated by execution"); + }); + + it("applies exactly the content that was presented, byte for byte", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + let presented = null; + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + confirmApply: async (_message, clusters) => { + presented = clusters[0].mergedContent; + return true; + }, + }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.applied.length, 1); + const survivor = store.rows.find((r) => r.id === result.applied[0].survivorId); + assert.equal(survivor.text, presented.abstract, "the applied text must exactly match what was presented in the plan"); + }); +}); + +describe("memory consolidate: item 8 staleness guard", () => { + it("skips a cluster whose member row was mutated between plan build and execution, without partially applying it", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + const logs = []; + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + log: (msg) => logs.push(msg), + confirmApply: async () => { + // Simulate a concurrent writer mutating the second member's row + // in the window between plan build and the user's confirmation. + const row = store.rows.find((r) => r.id === store.rows[1].id); + row.metadata = JSON.stringify({ ...JSON.parse(row.metadata), l0_abstract: "mutated by someone else" }); + return true; + }, + }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.applied.length, 0, "a stale cluster must never be partially or fully applied"); + assert.equal(result.staleSkipped.length, 1); + assert.deepEqual(result.staleSkipped[0].memberIds.sort(), [store.rows[0].id, store.rows[1].id].sort()); + assert.ok(logs.some((l) => /stale/i.test(l)), "a per-cluster report line must explain the skip"); + + const survivorRow = store.rows.find((r) => r.id === store.rows[0].id); + assert.equal(survivorRow.text, "Coffee order: oat milk latte", "the untouched survivor candidate must not have been merged in"); + }); + + it("a mutated row that disappears entirely (deleted/moved out of scope) is also treated as stale, not crashed on", async () => { + const rows = twoMemberMergeRows(); + const store = makeFakeStore(rows); + const llm = mergeDeciderLlm(); + const disappearedId = rows[1].id; + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + confirmApply: async () => { + const idx = store.rows.findIndex((r) => r.id === disappearedId); + store.rows.splice(idx, 1); + return true; + }, + }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(result.applied.length, 0); + assert.equal(result.staleSkipped.length, 1); + }); + + it("only skips the stale cluster, still applies unrelated fresh clusters in the same run", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts + 1 }), + makeRow({ abstract: "Desk setup: standing desk", factKey: "preferences:desk setup", vector: [0, 1, 0, 0], timestamp: ts + 2 }), + makeRow({ abstract: "Desk setup: standing desk, oak top", factKey: "preferences:desk setup", vector: [0, 1, 0, 0], timestamp: ts + 3 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "coffee dup" }, + { cluster_index: 2, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "desk dup" }, + ], + }; + } + return { + results: [ + { index: 1, abstract: "merged", overview: "", content: "merged" }, + { index: 2, abstract: "merged", overview: "", content: "merged" }, + ], + }; + }; + + const result = await runConsolidate( + { + ...store, + completeJson, + confirmApply: async () => { + // Mutate only the coffee cluster's second row. + const row = store.rows.find((r) => r.id === rows[1].id); + row.metadata = JSON.stringify({ ...JSON.parse(row.metadata), l0_abstract: "mutated" }); + return true; + }, + }, + { scope: "global", apply: false, autoConfirm: true, now: ts + 100_000 }, + ); + + assert.equal(result.staleSkipped.length, 1); + assert.equal(result.applied.length, 1, "the desk cluster must still apply despite the coffee cluster going stale"); + assert.equal(result.applied[0].survivorId, rows[2].id); + }); +}); + +describe("memory consolidate: item 8 direct --apply path (unchanged semantics)", () => { + it("gate -> build plan -> execute immediately, with no confirmApply call at all", async () => { + const store = makeFakeStore(twoMemberMergeRows()); + const llm = mergeDeciderLlm(); + let confirmApplyCalls = 0; + + const result = await runConsolidate( + { + ...store, + completeJson: llm.completeJson, + confirmApply: async () => { + confirmApplyCalls += 1; + return true; + }, + }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 }, + ); + + assert.equal(confirmApplyCalls, 0, "direct --apply must never call confirmApply"); + assert.equal(result.executed, true); + assert.equal(result.applied.length, 1); + assert.equal(result.applied[0].survivorId, store.rows[0].id); + }); +}); + +// --------------------------------------------------------------------------- +// Batched merge writer: one consolidate-merge-batch call per plan build +// --------------------------------------------------------------------------- + +/** + * N same-fact pairs, each pair on its own one-hot vector axis AND with + * fully pair-unique topic tokens (so neither cosine, fact_key, nor the + * token-overlap fallbacks can chain different pairs), so clustering + * yields exactly N units. + */ +function pairRows(pairCount) { + const ts = 1_700_000_000_000; + const rows = []; + for (let p = 0; p < pairCount; p++) { + const vector = Array.from({ length: pairCount }, (_, d) => (d === p ? 1 : 0)); + rows.push( + makeRow({ abstract: `topic${p + 1}key: value${p + 1}base`, factKey: `preferences:topic${p + 1}key`, vector, timestamp: ts + p * 10 }), + makeRow({ abstract: `topic${p + 1}key: value${p + 1}base extra${p + 1}note`, factKey: `preferences:topic${p + 1}key`, vector, timestamp: ts + p * 10 + 1 }), + ); + } + return rows; +} + +function batchWriterLlm({ verdictCount, onMergeBatch }) { + const mergeBatchCalls = []; + const calls = []; + const completeJson = async (prompt, label, system) => { + calls.push(label); + if (label === "consolidate-decide") { + return { + verdicts: Array.from({ length: verdictCount }, (_, i) => ({ + cluster_index: i + 1, + verdict: "merge", + survivor_index: 1, + absorbed_indices: [2], + reason: "duplicate pair", + })), + }; + } + if (label === "consolidate-merge-batch") { + mergeBatchCalls.push({ prompt, system }); + if (!onMergeBatch) throw new Error("unexpected consolidate-merge-batch call"); + return onMergeBatch(prompt, mergeBatchCalls.length); + } + throw new Error(`unexpected label: ${label}`); + }; + return { completeJson, mergeBatchCalls, calls }; +} + +function mergedResults(count, tag = "") { + return { + results: Array.from({ length: count }, (_, i) => ({ + index: i + 1, + abstract: `merged-${tag}${i + 1}`, + overview: "o", + content: "c", + })), + }; +} + +describe("memory consolidate: batched merge writer", () => { + const NOW = 1_700_100_000_000; + + it("writes every merge verdict's plan content with exactly one LLM call", async () => { + const store = makeFakeStore(pairRows(3)); + const llm = batchWriterLlm({ verdictCount: 3, onMergeBatch: () => mergedResults(3) }); + + const result = await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(llm.mergeBatchCalls.length, 1, "3 merge verdicts must share one batched merge-content call"); + assert.equal(llm.calls.filter((l) => l === "consolidate-merge").length, 0, "no per-verdict merge calls remain"); + const merged = result.clusters.filter((c) => c.action === "merge").map((c) => c.mergedContent?.abstract).sort(); + assert.deepEqual(merged, ["merged-1", "merged-2", "merged-3"]); + }); + + it("uses the batch shape even for a single merge verdict", async () => { + const store = makeFakeStore(pairRows(1)); + const llm = batchWriterLlm({ verdictCount: 1, onMergeBatch: () => mergedResults(1) }); + + await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(llm.mergeBatchCalls.length, 1); + assert.match(llm.mergeBatchCalls[0].prompt, /(^|\n)1\. Category: preferences/); + }); + + it("makes zero merge-writer calls when no verdict is a merge", async () => { + const store = makeFakeStore(pairRows(2)); + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "skip", reason: "unrelated" }, + { cluster_index: 2, verdict: "skip", reason: "unrelated" }, + ], + }; + } + throw new Error(`unexpected label: ${label}`); + }; + + const result = await runConsolidate( + { ...store, completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(result.clusters.filter((c) => c.action).length, 0); + }); + + it("degrades only the missing job to the survivor's own content, like a failed single-call fold", async () => { + const store = makeFakeStore(pairRows(2)); + const llm = batchWriterLlm({ + verdictCount: 2, + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged-1", overview: "o", content: "c" }] }), + }); + + const result = await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(llm.mergeBatchCalls.length, 1, "a malformed row must not fan out into extra calls"); + const mergeClusters = result.clusters.filter((c) => c.action === "merge"); + assert.equal(mergeClusters.length, 2, "both verdicts stay actionable"); + const abstracts = mergeClusters.map((c) => c.mergedContent?.abstract).sort(); + assert.ok(abstracts.includes("merged-1"), "the parsed job keeps its generated content"); + assert.ok( + abstracts.some((a) => /^topic\d+key: value\d+base$/.test(a)), + "the missing job falls back to its survivor's own content", + ); + }); + + it("falls back to survivor content for every job when the whole response is unparseable", async () => { + const store = makeFakeStore(pairRows(2)); + const llm = batchWriterLlm({ verdictCount: 2, onMergeBatch: () => null }); + + const result = await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(llm.mergeBatchCalls.length, 1); + const mergeClusters = result.clusters.filter((c) => c.action === "merge"); + assert.equal(mergeClusters.length, 2); + for (const cluster of mergeClusters) { + assert.match(cluster.mergedContent?.abstract, /^topic\d+key: value\d+base$/); + } + }); + + it("chunks oversized merge batches and covers every job exactly once", async () => { + const store = makeFakeStore(pairRows(12)); + const llm = batchWriterLlm({ + verdictCount: 12, + onMergeBatch: (_prompt, call) => mergedResults(call === 1 ? 10 : 2, `c${call}-`), + }); + + const result = await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + assert.equal(llm.mergeBatchCalls.length, 2, "12 merge verdicts over a cap of 10 must split into 2 calls"); + const merged = result.clusters.filter((c) => c.action === "merge").map((c) => c.mergedContent?.abstract); + assert.equal(merged.length, 12); + assert.equal(merged.filter((a) => /^merged-c1-/.test(a)).length, 10); + assert.equal(merged.filter((a) => /^merged-c2-/.test(a)).length, 2); + }); + + it("formats the batched merge prompt as numbered blocks without list markers", async () => { + const store = makeFakeStore(pairRows(2)); + const llm = batchWriterLlm({ verdictCount: 2, onMergeBatch: () => mergedResults(2) }); + + await runConsolidate( + { ...store, completeJson: llm.completeJson, autoConfirm: true, confirmApply: async () => false }, + { scope: "global", apply: false, autoConfirm: true, now: NOW }, + ); + + const { prompt } = llm.mergeBatchCalls[0]; + assert.match(prompt, /\n\n2\. Category: preferences/, "jobs are numbered inline and blank-line separated"); + assert.match(prompt, /^ {3}Existing memory:/m); + assert.match(prompt, /^ {3}New information/m); + assert.doesNotMatch(prompt, /^ *- (Abstract|Overview|Content)/m, "no leading list markers"); + }); +}); diff --git a/test/memory-consolidate.test.mjs b/test/memory-consolidate.test.mjs new file mode 100644 index 000000000..672a7eb46 --- /dev/null +++ b/test/memory-consolidate.test.mjs @@ -0,0 +1,1477 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; +import { Command } from "commander"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +const { + buildConsolidateCandidate, + clusterConsolidateCandidates, + chunkCluster, + parseConsolidateVerdict, + parseConsolidateBatchVerdicts, + runConsolidate, +} = jiti(path.join(testDir, "..", "src", "consolidate.ts")); + +const { buildConsolidatePrompt, buildConsolidateBatchPrompt } = jiti(path.join(testDir, "..", "src", "extraction-prompts.ts")); + +let nextId = 1; +function makeRow({ + scope = "global", + category = "preference", + memoryCategory = "preferences", + abstract, + overview = "", + content, + factKey, + source = "manual", + vector, + timestamp = 1_700_000_000_000, + invalidatedAt, + supersededBy, +}) { + // Zero-padded so lexicographic (string) sort matches insertion order -- + // item 3 sorts consolidate candidates by row id for determinism, and this + // keeps every existing fixture's insertion-order-based index assumptions + // (e.g. "row 4 is the reversal") valid under that sort. + const id = `row-${String(nextId++).padStart(6, "0")}`; + const metadata = { + l0_abstract: abstract, + l1_overview: overview, + l2_content: content || abstract, + memory_category: memoryCategory, + fact_key: factKey, + source, + valid_from: timestamp, + ...(invalidatedAt ? { invalidated_at: invalidatedAt } : {}), + ...(supersededBy ? { superseded_by: supersededBy } : {}), + }; + return { + id, + text: abstract, + vector, + category, + scope, + importance: 0.7, + timestamp, + metadata: JSON.stringify(metadata), + }; +} + +function makeFakeStore(initialRows) { + const rows = initialRows.map((r) => ({ ...r })); + return { + rows, + fetchRows: async (scopeFilter, maxTimestamp, limit) => { + return rows + .filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp) + .slice(0, limit) + .map((r) => ({ ...r })); + }, + update: async (id, patch) => { + const row = rows.find((r) => r.id === id); + if (!row) return null; + if (patch.text !== undefined) row.text = patch.text; + if (patch.vector !== undefined) row.vector = patch.vector; + if (patch.metadata !== undefined) row.metadata = patch.metadata; + return { ...row }; + }, + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + delete: async (id) => { + const idx = rows.findIndex((r) => r.id === id); + if (idx === -1) return false; + rows.splice(idx, 1); + return true; + }, + embed: async (text) => [text.length, 0, 0], + }; +} + +describe("memory consolidate: clustering", () => { + it("clusters rows purely by cosine similarity when fact_key is absent", () => { + const a = buildConsolidateCandidate(makeRow({ abstract: "Likes tea", vector: [1, 0, 0], factKey: undefined })); + const b = buildConsolidateCandidate(makeRow({ abstract: "Likes tea a lot", vector: [1, 0, 0], factKey: undefined })); + const c = buildConsolidateCandidate(makeRow({ abstract: "Unrelated fact", vector: [0, 1, 0], factKey: undefined })); + + const clusters = clusterConsolidateCandidates([a, b, c], 0.9); + assert.equal(clusters.length, 1); + assert.deepEqual(clusters[0].slice().sort(), [0, 1]); + }); + + it("clusters a low-cosine reversal row with its originals via a shared fact_key", () => { + const fk = "preferences:evening drink preference"; + const dup1 = buildConsolidateCandidate(makeRow({ abstract: "Evening drink: likes chamomile tea", vector: [1, 0, 0], factKey: fk })); + const dup2 = buildConsolidateCandidate(makeRow({ abstract: "Evening drink: likes chamomile tea", vector: [1, 0, 0], factKey: fk })); + const reversal = buildConsolidateCandidate(makeRow({ abstract: "Evening drink: quit chamomile tea", vector: [0, 0, 1], factKey: fk })); + const control = buildConsolidateCandidate(makeRow({ abstract: "Unrelated: prefers dark mode", vector: [0, 1, 0], factKey: "preferences:unrelated" })); + + const clusters = clusterConsolidateCandidates([dup1, dup2, reversal, control], 0.9); + assert.equal(clusters.length, 1, "exactly one cluster should form"); + assert.deepEqual(clusters[0].slice().sort(), [0, 1, 2], "the reversal row must join its originals despite low cosine similarity"); + }); + + it("never clusters a single unrelated row", () => { + const a = buildConsolidateCandidate(makeRow({ abstract: "Solo fact", vector: [1, 0, 0], factKey: "preferences:solo" })); + const clusters = clusterConsolidateCandidates([a], 0.9); + assert.equal(clusters.length, 0); + }); + + it("links a naturally-phrased reversal to a free-text reflection-mapped duplicate, even with a mismatched derived fact_key and low cosine", () => { + // Realistic cross-lane wording: smart-extraction follows the strict + // "[Merge key]: Description" abstract convention and gets a clean derived + // fact_key. Reflection writer-1 mapped rows carry NO stored fact_key and + // are free-text LLM summaries with no colon convention at all, so their + // DERIVED fact_key is the whole normalized sentence -- it will never match + // the smart-extraction row's key. A naturally-phrased reversal is in the + // same boat (confirmed against the real deriveFactKey, not a hypothetical). + const original = buildConsolidateCandidate( + makeRow({ + abstract: "Favorite soda: Coca-Cola", + vector: [1, 0, 0, 0], + factKey: "preferences:favorite soda", + source: "auto-capture", + }) + ); + const mappedDuplicate = buildConsolidateCandidate( + makeRow({ + abstract: "User prefers Coca-Cola as their favorite soft drink", + vector: [1, 0, 0, 0], + factKey: undefined, + source: "reflection", + }) + ); + const reversal = buildConsolidateCandidate( + makeRow({ + // Deliberately low cosine (orthogonal vector) to simulate an embedder + // that separates the reversal from its originals, and a free-text + // wording whose derived fact_key ("preferences:user has stopped + // drinking coca-cola") does not match "preferences:favorite soda". + abstract: "User has stopped drinking Coca-Cola", + vector: [0, 0, 0, 1], + factKey: undefined, + source: "manual", + }) + ); + const control = buildConsolidateCandidate( + makeRow({ + abstract: "User no longer works at Acme Corp", + vector: [0, 1, 0, 0], + factKey: undefined, + source: "manual", + }) + ); + + const clusters = clusterConsolidateCandidates([original, mappedDuplicate, reversal, control], 0.86); + assert.equal(clusters.length, 1, "exactly one cluster should form"); + assert.deepEqual( + clusters[0].slice().sort(), + [0, 1, 2], + "the reversal must join the cluster despite a mismatched derived fact_key and low cosine; the unrelated reversal-shaped control row must stay out" + ); + }); + + it("does not transitively chain two unrelated near-duplicate pairs together through a moderately-similar bridge pair", () => { + // Live dry-run found 8-row grab-bag clusters mixing weekly planning, + // standing desks, and roleplay notes -- none of these rows are + // reversal-shaped, so this is pure cosine transitivity chaining: + // A1~A2 direct link, A2~B1 direct link (the "bridge"), B1~B2 direct link, + // so union-find would glue all four into one cluster even though A1/A2 + // are never directly similar enough to B1/B2. Cosine values here are + // computed exactly (15/25/40/45-degree unit vectors), not guessed: + // A1-A2=0.966, A2-B1=0.906 (the bridge, well above 0.86), B1-B2=0.996, + // A1-B1=0.766, A1-B2=0.707 (both well below 0.86). + const A1 = buildConsolidateCandidate(makeRow({ abstract: "Prefers Sunday evening weekly planning.", vector: [1, 0], factKey: undefined })); + const A2 = buildConsolidateCandidate(makeRow({ abstract: "User now does weekly planning on Sunday evenings.", vector: [0.9659258262890683, 0.25881904510252074], factKey: undefined })); + const B1 = buildConsolidateCandidate(makeRow({ abstract: "Experimenting this month with a standing desk for back comfort.", vector: [0.766044443118978, 0.6427876096865393], factKey: undefined })); + const B2 = buildConsolidateCandidate(makeRow({ abstract: "Testing a standing desk setup this month to help with back pain.", vector: [0.7071067811865476, 0.7071067811865475], factKey: undefined })); + + const clusters = clusterConsolidateCandidates([A1, A2, B1, B2], 0.86); + assert.equal(clusters.length, 2, "the weekly-planning pair and the standing-desk pair must stay as two separate clusters"); + const sorted = clusters.map((c) => c.slice().sort()).sort((x, y) => x[0] - y[0]); + assert.deepEqual(sorted, [[0, 1], [2, 3]]); + }); + + it("does not let a long multi-topic reversal narrative bridge a tight cola cluster to unrelated desk rows (paraphrased live shape)", () => { + // Paraphrased from the live cluster-4 grab bag: a tight favorite-drink + + // reversal pair should stay together, but a long narrative row that also + // happens to mention "quit" (reversal-shaped) and touches several other + // topics at once must not bridge in the unrelated desk-move rows via + // incidental keyword overlap. + const favorite = buildConsolidateCandidate( + makeRow({ abstract: "User's favorite drink is Coca-Cola.", vector: [1, 0, 0], factKey: "preferences:favorite drink" }) + ); + const reversalShort = buildConsolidateCandidate( + makeRow({ abstract: "User will no longer drink cola", vector: [0, 0, 1], factKey: undefined }) + ); + const longNarrative = buildConsolidateCandidate( + makeRow({ + abstract: + "User quit drinking Coca-Cola after the fridge explosion incident. Decided to redesign their room and moved their desk from a dark corner to next to the window for natural light and better productivity.", + vector: [0, 1, 0], + factKey: undefined, + }) + ); + const deskMove = buildConsolidateCandidate( + makeRow({ abstract: "User will move their desk to sit directly next to the window for natural light.", vector: [0, 1, 0], factKey: undefined }) + ); + + const clusters = clusterConsolidateCandidates([favorite, reversalShort, longNarrative, deskMove], 0.86); + + const colaCluster = clusters.find((c) => c.includes(0)); + assert.ok(colaCluster, "the favorite-drink row must be in some cluster"); + assert.ok(colaCluster.includes(1), "the short reversal must join the favorite-drink row"); + assert.ok( + !colaCluster.includes(3), + "the unrelated desk-move row must not be glued into the cola cluster through the long narrative row" + ); + }); + + it("clusters 3 plain near-duplicate rows from different write lanes with their contradiction, even though none of the duplicates is reversal-shaped (item 4 motivating fixture)", () => { + // Synthetic, cross-lane favorite-drink family: three PLAIN statements of + // the same fact, phrased the way three different write lanes would + // phrase it (strict colon convention, free-text reflection-mapped + // prose, and a casual auto-capture paraphrase) -- none contains reversal + // wording, so the pre-item-4 reversal-gated topic-overlap fallback never + // links them to EACH OTHER (only ever to the reversal row, and only for + // whichever one becomes reachable first via seed order). Deliberately + // orthogonal vectors and mismatched fact_keys simulate real cross-lane + // embedding/tokenization drift, so cosine and fact_key both miss too. + const strictConvention = buildConsolidateCandidate( + makeRow({ abstract: "Favorite drink: cola", vector: [1, 0, 0, 0], factKey: "preferences:favorite drink", source: "manual" }) + ); + const freeTextMapped = buildConsolidateCandidate( + makeRow({ abstract: "The user really likes cola as their favorite drink", vector: [0, 1, 0, 0], factKey: undefined, source: "reflection" }) + ); + const casualParaphrase = buildConsolidateCandidate( + makeRow({ abstract: "Cola is what gets ordered most evenings", vector: [0, 0, 1, 0], factKey: undefined, source: "auto-capture" }) + ); + const contradiction = buildConsolidateCandidate( + makeRow({ abstract: "No longer drinks cola", vector: [0, 0, 0, 1], factKey: undefined, source: "manual" }) + ); + const unrelated = buildConsolidateCandidate( + makeRow({ abstract: "Prefers a standing desk for back comfort", vector: [1, 1, 0, 0], factKey: "preferences:desk setup", source: "manual" }) + ); + + const clusters = clusterConsolidateCandidates( + [strictConvention, freeTextMapped, casualParaphrase, contradiction, unrelated], + 0.86 + ); + + assert.equal(clusters.length, 1, "exactly one cluster should form for the favorite-drink family"); + assert.deepEqual( + clusters[0].slice().sort(), + [0, 1, 2, 3], + "all 3 cross-lane duplicates and the contradiction must land in the SAME cluster; the unrelated desk row must stay out" + ); + }); +}); + +describe("memory consolidate: cluster chunking", () => { + it("chunks a cluster into groups no larger than the cap", () => { + const indices = Array.from({ length: 10 }, (_, i) => i); + const chunks = chunkCluster(indices, 8); + assert.equal(chunks.length, 2); + assert.equal(chunks[0].length, 8); + assert.equal(chunks[1].length, 2); + }); +}); + +describe("memory consolidate: verdict parsing", () => { + it("accepts a well-formed skip verdict", () => { + const verdict = parseConsolidateVerdict({ verdict: "skip", reason: "distinct facts" }, 3); + assert.deepEqual(verdict, { verdict: "skip", reason: "distinct facts" }); + }); + + it("accepts a well-formed merge verdict with survivor and absorbed indices", () => { + const verdict = parseConsolidateVerdict( + { verdict: "merge", survivor_index: 1, absorbed_indices: [2, 3], reason: "duplicates" }, + 3 + ); + assert.deepEqual(verdict, { verdict: "merge", survivorIndex: 1, absorbedIndices: [2, 3], reason: "duplicates" }); + }); + + it("rejects an unknown verdict string", () => { + assert.equal(parseConsolidateVerdict({ verdict: "delete_everything" }, 3), null); + }); + + it("rejects merge missing absorbed_indices", () => { + assert.equal(parseConsolidateVerdict({ verdict: "merge", survivor_index: 1 }, 3), null); + }); + + it("rejects an out-of-range survivor_index", () => { + assert.equal(parseConsolidateVerdict({ verdict: "merge", survivor_index: 9, absorbed_indices: [1] }, 3), null); + }); + + it("rejects non-object input", () => { + assert.equal(parseConsolidateVerdict(null, 3), null); + assert.equal(parseConsolidateVerdict("skip", 3), null); + }); +}); + +describe("memory consolidate: prompt shape", () => { + it("returns a {system, user} split prompt naming the four verdicts", () => { + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }, + { index: 2, category: "preferences", abstract: "b", overview: "", content: "b", source: "reflection" }, + ]); + assert.equal(typeof prompt.system, "string"); + assert.equal(typeof prompt.user, "string"); + assert.match(prompt.system, /you are a memory consolidation decider/i); + for (const verb of ["skip", "merge", "supersede", "contradict"]) { + assert.match(prompt.system, new RegExp(verb, "i")); + } + assert.match(prompt.user, /1\./); + assert.match(prompt.user, /2\./); + }); + + it("tells the decider that supersede is non-destructive soft-invalidation, not deletion", () => { + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }, + ]); + // Defect 1: the live decider reasoned "kept as historical rather than + // REQUIRING DELETION" as a reason to skip a clear reversal, because the + // prompt never said supersede preserves history. Mirror buildDedupPrompt's + // own SUPERSEDE language ("kept as historical but no longer current"). + assert.match(prompt.system, /not.{0,60}destructive/i); + assert.match(prompt.system, /never (be )?delet/i); + assert.match(prompt.system, /historical/i); + }); + + it("tells the decider it may act on a subset of the cluster, leaving append-only rows untouched", () => { + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }, + ]); + // Defect 2: the live decider skipped whole 3-8 row clusters just because + // ONE member was an append-only events/cases row, even when the rest were + // exact duplicates. survivor_index/absorbed_indices already support acting + // on a subset (unlisted rows are simply left untouched) -- the prompt must + // say so explicitly instead of implying every row must be covered. + assert.match(prompt.system, /not.{0,40}(need|have) to (act on|cover) every row|leave.{0,40}(out|untouched)/i); + assert.match(prompt.system, /append-only/i); + }); + + it("adds a one-line source legend to the decider system prompt so provenance informs survivor choice", () => { + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }, + ]); + assert.match(prompt.system, /legacy\s*=\s*pre-smart-format rows/i); + assert.match(prompt.system, /manual\s*=\s*operator memory_store saves/i); + assert.match(prompt.system, /auto-capture\s*=\s*extraction lane/i); + assert.match(prompt.system, /reflection\*\s*=\s*mirror lanes/i); + assert.match(prompt.system, /manual rows are operator-authored and strong survivor candidates/i); + }); + + it("includes each member's timestamp, and valid_from only when it differs, so supersede recency is explicit rather than inferred from text", () => { + const ts1 = 1_700_000_000_000; + const ts2 = 1_700_100_000_000; + const vf2 = 1_699_000_000_000; + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual", timestamp: ts1, validFrom: ts1 }, + { index: 2, category: "preferences", abstract: "b", overview: "", content: "b", source: "auto-capture", timestamp: ts2, validFrom: vf2 }, + ]); + assert.ok( + prompt.user.includes(`timestamp: ${new Date(ts1).toISOString()}`), + "member 1's timestamp must appear in the listing" + ); + assert.ok( + prompt.user.includes(`timestamp: ${new Date(ts2).toISOString()}`), + "member 2's timestamp must appear in the listing" + ); + assert.ok( + prompt.user.includes(`valid_from: ${new Date(vf2).toISOString()}`), + "member 2's valid_from differs from its timestamp and must be shown explicitly" + ); + assert.ok( + !prompt.user.includes(`valid_from: ${new Date(ts1).toISOString()}`), + "member 1's valid_from equals its timestamp and must not be printed redundantly" + ); + }); + + it("tells the decider same-category append-only duplicates may be merged, but never superseded", () => { + const prompt = buildConsolidatePrompt([ + { index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }, + ]); + assert.match(prompt.system, /append-only/i); + assert.match(prompt.system, /same append-only category/i); + assert.match(prompt.system, /can never be superseded or contradicted/i); + }); + + it("renders identical L0/L1/L2 tiers once per member instead of repeating the same raw fallback text three times", () => { + // mapped/manual/legacy rows without real overview/content commonly fall + // back to the raw abstract text in all three tiers (see + // src/smart-metadata.ts's parseSmartMetadata: l2_content falls back to + // raw text, l1_overview falls back to `- ${abstract}`) -- printing that + // fact three times per member wastes cluster-listing space for no signal. + const thin = { + index: 1, + category: "preferences", + abstract: "Likes tea", + overview: "- Likes tea", + content: "Likes tea", + source: "legacy", + }; + const rich = { + index: 2, + category: "preferences", + abstract: "Coffee order: oat milk latte", + overview: "## Preference\n- oat milk latte", + content: "User always orders an oat milk latte with extra foam.", + source: "manual", + }; + const prompt = buildConsolidatePrompt([thin, rich]); + + assert.ok(prompt.user.includes("Fact: Likes tea"), "thin member collapses to a single Fact: line"); + assert.ok(!/Abstract: Likes tea/.test(prompt.user), "thin member must not repeat the abstract label"); + assert.ok(!/Overview: - Likes tea/.test(prompt.user), "thin member must not repeat the overview label"); + + assert.ok( + prompt.user.includes("Abstract: Coffee order: oat milk latte"), + "rich member with genuinely distinct tiers keeps the full Abstract/Overview/Content rendering" + ); + assert.ok(prompt.user.includes("User always orders an oat milk latte with extra foam.")); + }); +}); + +describe("memory consolidate: batch prompt shape", () => { + function member(index, abstract) { + return { index, category: "preferences", abstract, overview: "", content: abstract, source: "manual" }; + } + + it("returns a {system, user} split prompt asking for a JSON array of verdicts tagged with cluster_index", () => { + const prompt = buildConsolidateBatchPrompt([ + { clusterIndex: 1, members: [member(1, "a"), member(2, "b")] }, + { clusterIndex: 2, members: [member(1, "c"), member(2, "d")] }, + ]); + assert.equal(typeof prompt.system, "string"); + assert.equal(typeof prompt.user, "string"); + assert.match(prompt.system, /you are a memory consolidation decider/i); + assert.match(prompt.system, /verdicts/i); + assert.match(prompt.system, /cluster_index/i); + for (const verb of ["skip", "merge", "supersede", "contradict"]) { + assert.match(prompt.system, new RegExp(verb, "i")); + } + }); + + it("lists every cluster in the user prompt, each with its own 1-based member numbering", () => { + const prompt = buildConsolidateBatchPrompt([ + { clusterIndex: 1, members: [member(1, "first cluster row one"), member(2, "first cluster row two")] }, + { clusterIndex: 2, members: [member(1, "second cluster row one")] }, + ]); + assert.match(prompt.user, /cluster 1/i); + assert.match(prompt.user, /cluster 2/i); + assert.ok(prompt.user.includes("first cluster row one")); + assert.ok(prompt.user.includes("first cluster row two")); + assert.ok(prompt.user.includes("second cluster row one")); + }); + + it("still tells the decider that supersede is non-destructive and clusters may be decided as a subset", () => { + const prompt = buildConsolidateBatchPrompt([{ clusterIndex: 1, members: [member(1, "a")] }]); + assert.match(prompt.system, /not.{0,60}destructive/i); + assert.match(prompt.system, /never (be )?delet/i); + assert.match(prompt.system, /historical/i); + assert.match(prompt.system, /append-only/i); + }); + + it("still includes the source legend and per-member timestamps", () => { + const ts = 1_700_000_000_000; + const prompt = buildConsolidateBatchPrompt([ + { clusterIndex: 1, members: [{ ...member(1, "a"), timestamp: ts }] }, + ]); + assert.match(prompt.system, /legacy\s*=\s*pre-smart-format rows/i); + assert.ok(prompt.user.includes(`timestamp: ${new Date(ts).toISOString()}`)); + }); + + it("still tells the decider same-category append-only duplicates may be merged, but never superseded", () => { + const prompt = buildConsolidateBatchPrompt([{ clusterIndex: 1, members: [member(1, "a")] }]); + assert.match(prompt.system, /append-only/i); + assert.match(prompt.system, /same append-only category/i); + assert.match(prompt.system, /can never be superseded or contradicted/i); + }); +}); + +// Prompt-architecture slot conformance: every static block (identity, task +// framing, rules, the JSON output contract) lives in the SYSTEM slot; the +// USER slot carries only the numbered cluster/job data. Both consolidate +// prompts already deliver a real split (completeJson(user, label, system)), +// so these pins keep static text from ever drifting into the user slot. +describe("memory consolidate: batched prompt slot conformance", () => { + const { buildConsolidateBatchMergePrompt } = jiti(path.join(testDir, "..", "src", "extraction-prompts.ts")); + + function assertSlotSplit({ system, user }, { staticSentinels, userOpener }) { + for (const sentinel of staticSentinels) { + assert.ok(system.includes(sentinel), `system must carry static sentinel: ${sentinel}`); + assert.ok(!user.includes(sentinel), `user must NOT carry static sentinel: ${sentinel}`); + } + assert.ok(user.startsWith(userOpener), `user must open with the data header ${JSON.stringify(userOpener)}`); + } + + it("consolidate-decide: identity, verdict rules, and output contract are system-only", () => { + const prompt = buildConsolidateBatchPrompt([ + { + clusterIndex: 1, + members: [ + { index: 1, category: "preferences", abstract: "row one", overview: "", content: "row one", source: "manual" }, + ], + }, + ]); + assertSlotSplit(prompt, { + staticSentinels: [ + "You are a memory consolidation decider.", + "Decision criteria: apply these checks in order", + "Return JSON only:", + "Source legend:", + ], + userOpener: "Cluster 1 members:", + }); + assert.ok(prompt.user.includes("row one"), "member rows are per-call data and belong in user"); + assert.ok(!prompt.system.includes("row one")); + }); + + it("consolidate-merge-batch: identity, merge requirements, and output contract are system-only", () => { + const prompt = buildConsolidateBatchMergePrompt([ + { + category: "preferences", + existing: { abstract: "existing abstract", overview: "existing overview", content: "existing content" }, + additions: [{ abstract: "new abstract", overview: "new overview", content: "new content" }], + }, + ]); + assertSlotSplit(prompt, { + staticSentinels: [ + "You are a memory consolidation merge writer.", + "Requirements:", + "Return JSON only, with exactly one entry per job", + ], + userOpener: "Merge jobs:", + }); + assert.match(prompt.user, /1\. Category: preferences/); + assert.ok(prompt.user.includes("existing abstract")); + assert.ok(prompt.user.includes("new content")); + assert.ok(!prompt.system.includes("existing abstract"), "job payloads must never leak into system"); + }); +}); + +describe("memory consolidate: batch verdict parsing", () => { + it("parses multiple well-formed verdicts keyed by cluster_index", () => { + const raw = { + verdicts: [ + { cluster_index: 1, verdict: "skip", reason: "distinct" }, + { cluster_index: 2, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }, + ], + }; + const units = [ + { clusterIndex: 1, memberCount: 2 }, + { clusterIndex: 2, memberCount: 2 }, + ]; + const map = parseConsolidateBatchVerdicts(raw, units); + assert.equal(map.size, 2); + assert.deepEqual(map.get(1), { verdict: "skip", reason: "distinct" }); + assert.deepEqual(map.get(2), { verdict: "merge", survivorIndex: 1, absorbedIndices: [2], reason: "dup" }); + }); + + it("fails closed per-cluster: a malformed entry for one cluster does not affect other clusters' valid verdicts", () => { + const raw = { + verdicts: [ + { cluster_index: 1, verdict: "bogus_verdict" }, + { cluster_index: 2, verdict: "skip", reason: "fine" }, + ], + }; + const units = [ + { clusterIndex: 1, memberCount: 2 }, + { clusterIndex: 2, memberCount: 2 }, + ]; + const map = parseConsolidateBatchVerdicts(raw, units); + assert.equal(map.has(1), false, "malformed entry must not produce a verdict for cluster 1"); + assert.equal(map.has(2), true, "cluster 2's valid verdict must still parse"); + assert.equal(map.get(2).verdict, "skip"); + }); + + it("ignores an entry whose cluster_index does not match any known unit", () => { + const raw = { verdicts: [{ cluster_index: 99, verdict: "skip", reason: "orphan" }] }; + const units = [{ clusterIndex: 1, memberCount: 2 }]; + const map = parseConsolidateBatchVerdicts(raw, units); + assert.equal(map.size, 0); + }); + + it("keeps the first entry and ignores later duplicates for the same cluster_index", () => { + const raw = { + verdicts: [ + { cluster_index: 1, verdict: "skip", reason: "first" }, + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "second" }, + ], + }; + const units = [{ clusterIndex: 1, memberCount: 2 }]; + const map = parseConsolidateBatchVerdicts(raw, units); + assert.equal(map.size, 1); + assert.equal(map.get(1).verdict, "skip"); + assert.equal(map.get(1).reason, "first"); + }); + + it("returns an empty map when the whole response is not a {verdicts: [...]} shape", () => { + const units = [{ clusterIndex: 1, memberCount: 2 }]; + assert.equal(parseConsolidateBatchVerdicts(null, units).size, 0); + assert.equal(parseConsolidateBatchVerdicts({ nonsense: true }, units).size, 0); + assert.equal(parseConsolidateBatchVerdicts({ verdicts: "not-an-array" }, units).size, 0); + }); +}); + +describe("memory consolidate: batch prompt rubric tightening", () => { + it("states explicit decision criteria for each verdict, not just a one-line description", () => { + const prompt = buildConsolidateBatchPrompt([ + { clusterIndex: 1, members: [{ index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }] }, + ]); + assert.match(prompt.system, /decision criteria/i); + }); + + it("tells the decider to prefer supersede over merge when the choice is ambiguous", () => { + const prompt = buildConsolidateBatchPrompt([ + { clusterIndex: 1, members: [{ index: 1, category: "preferences", abstract: "a", overview: "", content: "a", source: "manual" }] }, + ]); + assert.match(prompt.system, /ambiguous/i); + assert.match(prompt.system, /prefer supersede/i); + }); +}); + +describe("memory consolidate: deterministic verdicts", () => { + it("passes temperature 0 for the batched consolidate-decide call", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1 }), + ]; + const store = makeFakeStore(rows); + let capturedTemperature; + const completeJson = async (_prompt, label, _system, temperature) => { + if (label === "consolidate-decide") { + capturedTemperature = temperature; + return { verdicts: [{ cluster_index: 1, verdict: "skip", reason: "no action" }] }; + } + return null; + }; + + await runConsolidate({ ...store, completeJson }, { scope: "global", apply: false, autoConfirm: true, now: ts + 1000 }); + + assert.equal(capturedTemperature, 0, "the consolidate-decide call must request temperature 0"); + }); + + it("produces byte-identical prompt text for the same candidate set regardless of fetch order", async () => { + const ts = 1_700_000_000_000; + const rowsInOrder = [ + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1 }), + makeRow({ abstract: "Tea order: chamomile", factKey: "preferences:tea order", vector: [0, 1], timestamp: ts + 2 }), + makeRow({ abstract: "Tea order: chamomile, no sugar", factKey: "preferences:tea order", vector: [0, 1], timestamp: ts + 3 }), + ]; + const rowsShuffled = [rowsInOrder[3], rowsInOrder[1], rowsInOrder[0], rowsInOrder[2]]; + + const capturedPrompts = []; + const completeJson = async (prompt, label) => { + if (label === "consolidate-decide") { + capturedPrompts.push(prompt); + return { + verdicts: [ + { cluster_index: 1, verdict: "skip", reason: "n/a" }, + { cluster_index: 2, verdict: "skip", reason: "n/a" }, + ], + }; + } + return null; + }; + + await runConsolidate( + { ...makeFakeStore(rowsInOrder), completeJson }, + { scope: "global", apply: false, autoConfirm: true, now: ts + 1000 } + ); + await runConsolidate( + { ...makeFakeStore(rowsShuffled), completeJson }, + { scope: "global", apply: false, autoConfirm: true, now: ts + 1000 } + ); + + assert.equal(capturedPrompts.length, 2); + assert.equal( + capturedPrompts[0], + capturedPrompts[1], + "prompt text must be byte-identical regardless of the order fetchRows returns the same candidate set in" + ); + }); + + it("acceptance: 3 consecutive dry-runs on an unchanged store produce byte-identical verdict sets", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Coffee order: oat milk latte", content: "x", factKey: "preferences:coffee order", vector: [1, 0, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", content: "y", factKey: "preferences:coffee order", vector: [1, 0, 0], timestamp: ts + 1 }), + makeRow({ abstract: "Desk setup: standing desk", content: "z", factKey: "preferences:desk setup", vector: [0, 1, 0], timestamp: ts + 2 }), + makeRow({ abstract: "Desk setup: standing desk, oak top", content: "w", factKey: "preferences:desk setup", vector: [0, 1, 0], timestamp: ts + 3 }), + ]; + + // A deterministic stand-in for a temperature-0 LLM: a pure function of + // the prompt text itself, so if the prompt is byte-identical across + // runs (guaranteed by the sort-before-build fix) the "model" output is + // byte-identical too -- this isolates OUR code's contribution to + // determinism from real model non-determinism, which a unit test can't + // exercise directly. + const completeJson = async (prompt, label) => { + if (label !== "consolidate-decide") return null; + const clusterCount = (prompt.match(/^Cluster \d+ members:/gm) || []).length; + const verdicts = []; + for (let i = 1; i <= clusterCount; i++) { + verdicts.push({ cluster_index: i, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: `deterministic-${i}` }); + } + return { verdicts }; + }; + + const results = []; + for (let i = 0; i < 3; i++) { + const store = makeFakeStore(rows); + results.push(await runConsolidate({ ...store, completeJson }, { scope: "global", apply: false, autoConfirm: true, now: ts + 100_000 })); + } + + assert.deepEqual(results[0].clusters, results[1].clusters, "run 1 vs run 2 verdict sets must be byte-identical"); + assert.deepEqual(results[1].clusters, results[2].clusters, "run 2 vs run 3 verdict sets must be byte-identical"); + }); +}); + +describe("memory consolidate: orchestration", () => { + function buildFixtureRows() { + const fk = "preferences:evening drink preference"; + const ts = 1_700_000_000_000; + return [ + makeRow({ abstract: "Evening drink preference: likes chamomile tea", content: "User mentioned liking chamomile tea in the evening.", factKey: fk, source: "manual", vector: [1, 0, 0, 0], timestamp: ts }), + makeRow({ abstract: "Evening drink preference: likes chamomile tea", content: "Reflection noted the user's chamomile tea habit.", factKey: fk, source: "reflection", vector: [1, 0, 0, 0], timestamp: ts + 1000 }), + makeRow({ abstract: "Evening drink preference: likes chamomile tea", content: "Extracted from conversation about evening routines.", factKey: fk, source: "auto-capture", vector: [1, 0, 0, 0], timestamp: ts + 2000 }), + makeRow({ abstract: "Evening drink preference: quit chamomile tea", content: "User decided to stop drinking chamomile tea.", factKey: fk, source: "manual", vector: [0, 0, 0, 1], timestamp: ts + 3000 }), + makeRow({ abstract: "Editor theme preference: prefers dark mode", content: "User prefers a dark editor theme.", factKey: "preferences:editor theme preference", source: "manual", vector: [0, 1, 0, 0], timestamp: ts + 4000 }), + ]; + } + + it("dry-run clusters the four related rows and reports a supersede verdict, leaving the control row untouched", async () => { + const store = makeFakeStore(buildFixtureRows()); + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "supersede", + survivor_index: 4, + absorbed_indices: [1, 2, 3], + reason: "the reversal row supersedes the three duplicate rows", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson }, + { scope: "global", apply: false, autoConfirm: true, now: 1_700_100_000_000 } + ); + + assert.equal(result.apply, false); + assert.equal(result.scanned, 5); + assert.equal(result.eligible, 5); + assert.equal(result.clusters.length, 1); + assert.equal(result.clusters[0].memberIds.length, 4); + assert.equal(result.clusters[0].verdict.verdict, "supersede"); + assert.equal(result.applied.length, 0, "dry-run must not apply anything"); + assert.equal(store.rows.length, 5, "dry-run must not delete or mutate any row"); + }); + + it("apply mode executes the supersede verdict, invalidates the duplicates, and leaves the control row byte-identical", async () => { + const fixture = buildFixtureRows(); + const controlBefore = fixture.find((r) => r.text.includes("dark mode")); + const store = makeFakeStore(fixture); + const audits = []; + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "supersede", + survivor_index: 4, + absorbed_indices: [1, 2, 3], + reason: "the reversal row supersedes the three duplicate rows", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson, onAudit: (a) => audits.push(a) }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 } + ); + + assert.equal(result.applied.length, 1); + assert.equal(audits.length, 1); + + const survivorRow = store.rows.find((r) => r.text.includes("quit chamomile tea")); + assert.ok(survivorRow, "the reversal row must remain"); + const survivorMeta = JSON.parse(survivorRow.metadata); + assert.ok(survivorMeta.consolidation_audit, "survivor gets an audit trail"); + + const absorbedRows = store.rows.filter((r) => r.text.includes("likes chamomile tea")); + assert.equal(absorbedRows.length, 3, "absorbed rows are not deleted, only invalidated"); + for (const row of absorbedRows) { + const meta = JSON.parse(row.metadata); + assert.ok(meta.invalidated_at, "each absorbed row must be marked invalidated"); + assert.equal(meta.superseded_by, survivorRow.id); + } + + const controlAfter = store.rows.find((r) => r.text.includes("dark mode")); + assert.deepEqual(controlAfter, { ...controlBefore }, "control row must be byte-identical after apply"); + }); + + it("is idempotent: a second apply run over the same store makes zero further changes", async () => { + const store = makeFakeStore(buildFixtureRows()); + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "supersede", + survivor_index: 4, + absorbed_indices: [1, 2, 3], + reason: "reversal supersedes duplicates", + }, + ], + }); + + await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 }); + const secondResult = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: 1_700_200_000_000 }); + + assert.equal(secondResult.applied.length, 0, "no cluster should reform once duplicates are invalidated"); + }); + + it("executes a pure merge verdict by combining two duplicate rows into one via the merge-writer prompt", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Coffee order: oat milk latte", content: "User orders an oat milk latte.", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", content: "User specified extra hot as well.", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same fact, second row adds detail" }, + ], + }; + } + return { + results: [ + { index: 1, abstract: "Coffee order: oat milk latte, extra hot", overview: "", content: "User orders an oat milk latte, extra hot." }, + ], + }; + }; + + const result = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: ts + 100_000 }); + + assert.equal(result.applied.length, 1); + assert.equal(result.applied[0].survivorId, rows[0].id); + assert.deepEqual(result.applied[0].absorbedIds, [rows[1].id]); + + assert.equal(store.rows.length, 2, "merge is non-destructive: the absorbed row must still be present, only invalidated"); + const survivorRow = store.rows.find((r) => r.id === rows[0].id); + assert.equal(survivorRow.text, "Coffee order: oat milk latte, extra hot"); + + const absorbedRow = store.rows.find((r) => r.id === rows[1].id); + assert.ok(absorbedRow, "the absorbed row must not be hard-deleted"); + const absorbedMeta = JSON.parse(absorbedRow.metadata); + assert.ok(absorbedMeta.invalidated_at, "the absorbed row must be marked invalidated"); + assert.equal(absorbedMeta.superseded_by, rows[0].id, "the absorbed row must point at the survivor"); + assert.ok(absorbedMeta.consolidation_audit, "the absorbed row must carry its own consolidation audit"); + assert.equal(absorbedMeta.consolidation_audit.action, "merge"); + assert.equal(absorbedMeta.consolidation_audit.survivorId, rows[0].id); + }); + + it("skips a cluster with a warning when the LLM response is malformed, without failing the run", async () => { + const store = makeFakeStore(buildFixtureRows()); + const logs = []; + const completeJson = async () => ({ nonsense: true }); + + const result = await runConsolidate( + { ...store, completeJson, log: (msg) => logs.push(msg) }, + { scope: "global", apply: true, autoConfirm: true, now: 1_700_100_000_000 } + ); + + assert.equal(result.skippedMalformed, 1); + assert.equal(result.applied.length, 0); + assert.equal(store.rows.length, 5, "no rows touched when the verdict is malformed"); + assert.ok(logs.some((l) => /malformed|missing/i.test(l))); + }); + + it("excludes reflection-category rows by default and includes them with the opt-in flag", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "reflection", memoryCategory: "patterns", abstract: "Reflection slice: always verify output", factKey: undefined, vector: [1, 0], timestamp: ts }), + makeRow({ category: "reflection", memoryCategory: "patterns", abstract: "Reflection slice: always verify output twice", factKey: undefined, vector: [1, 0], timestamp: ts + 1 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async () => ({ + verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }], + }); + + const excluded = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: false, autoConfirm: true, now: ts + 100 }); + assert.equal(excluded.eligible, 0, "reflection rows excluded by default"); + + const included = await runConsolidate( + { ...store, completeJson }, + { scope: "global", apply: false, autoConfirm: true, now: ts + 100, includeReflectionSlices: true } + ); + assert.equal(included.eligible, 2, "reflection rows included with the opt-in flag"); + }); + + it("refuses to supersede append-only categories even if the LLM says to, even within the same category", async () => { + // Append-only means invalidation-protection, not merge-immunity: supersede + // (and contradict) invalidate the absorbed row's currency, which must never + // happen to an events/cases row regardless of whether every acted-upon row + // shares the same append-only category. + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1", factKey: "events:deploy", vector: [1, 0], timestamp: ts }), + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1 again", factKey: "events:deploy", vector: [1, 0], timestamp: ts + 1 }), + ]; + const store = makeFakeStore(rows); + const logs = []; + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "supersede", + survivor_index: 2, + absorbed_indices: [1], + reason: "an unsafe LLM verdict that must be rejected", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson, log: (msg) => logs.push(msg) }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100 } + ); + + assert.equal(result.applied.length, 0, "append-only categories must never be superseded, even on LLM instruction"); + assert.equal(store.rows.length, 2, "both events rows must remain untouched"); + assert.ok(logs.some((l) => /append-only/i.test(l))); + }); + + it("allows merging near-identical duplicate rows within the same append-only category", async () => { + // The append-only shield exists to protect against invalidation, not to + // block dedup of true duplicates -- two rows describing the exact same + // occurrence should still be able to merge into one. + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1", factKey: "events:deploy", vector: [1, 0], timestamp: ts }), + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1 again", factKey: "events:deploy", vector: [1, 0], timestamp: ts + 1 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "merge", + survivor_index: 1, + absorbed_indices: [2], + reason: "byte-near duplicate event rows describing the same deploy", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100 } + ); + + assert.equal(result.applied.length, 1, "same-category append-only duplicates must be allowed to merge"); + assert.equal(result.applied[0].survivorId, rows[0].id); + assert.deepEqual(result.applied[0].absorbedIds, [rows[1].id]); + assert.equal(store.rows.length, 2, "merge is non-destructive: both rows remain present"); + const absorbedRow = store.rows.find((r) => r.id === rows[1].id); + assert.ok( + JSON.parse(absorbedRow.metadata).invalidated_at, + "the absorbed duplicate must be marked invalidated, not hard-deleted" + ); + }); + + it("still refuses to merge append-only rows across different categories", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1", factKey: "events:deploy", vector: [1, 0], timestamp: ts }), + makeRow({ category: "fact", memoryCategory: "cases", abstract: "Deploy runbook: roll back to v0 on failure", factKey: "cases:deploy", vector: [1, 0], timestamp: ts + 1 }), + ]; + const store = makeFakeStore(rows); + const logs = []; + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "merge", + survivor_index: 1, + absorbed_indices: [2], + reason: "an unsafe cross-category LLM verdict that must be rejected", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson, log: (msg) => logs.push(msg) }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100 } + ); + + assert.equal(result.applied.length, 0, "append-only rows from different categories must never merge into each other"); + assert.equal(store.rows.length, 2); + assert.ok(logs.some((l) => /append-only/i.test(l))); + }); + + it("a preference row can never supersede an event row", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "preference", memoryCategory: "preferences", abstract: "Prefers the new deploy process", factKey: "preferences:deploy process", vector: [1, 0], timestamp: ts + 1 }), + makeRow({ category: "decision", memoryCategory: "events", abstract: "Deploy event: shipped v1", factKey: "events:deploy process", vector: [1, 0], timestamp: ts }), + ]; + const store = makeFakeStore(rows); + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "supersede", + survivor_index: 1, + absorbed_indices: [2], + reason: "an unsafe LLM verdict claiming the preference supersedes the event", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100 } + ); + + assert.equal(result.applied.length, 0, "a non-append-only row must never supersede an append-only row"); + assert.equal(store.rows.length, 2); + }); + + it("still merges the actionable duplicates in a cluster that also contains an unreferenced append-only row (paraphrased live shape)", async () => { + // Paraphrased from a live dry-run: a lamp-preference cluster of 3 rows + // where 2 are true preference duplicates and 1 is a "finalized decision" + // events row. The decider should be able to merge just the 2 duplicates, + // leaving the append-only row alone -- not skip the whole cluster. + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ category: "preference", memoryCategory: "preferences", abstract: "Reading lamp preference: warm white, bookshelf side.", factKey: "preferences:reading lamp", vector: [1, 0], timestamp: ts }), + makeRow({ category: "decision", memoryCategory: "events", abstract: "Reading lamp finalized: warm white, positioned on the bookshelf side.", factKey: "events:reading lamp", vector: [1, 0], timestamp: ts + 1 }), + makeRow({ category: "preference", memoryCategory: "preferences", abstract: "Prefers warm white lighting on the bookshelf side for reading.", factKey: "preferences:reading lamp", vector: [1, 0], timestamp: ts + 2 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async () => ({ + verdicts: [ + { + cluster_index: 1, + verdict: "merge", + survivor_index: 1, + absorbed_indices: [3], + reason: "rows 1 and 3 are the same lamp preference; row 2 is an append-only decision left untouched", + }, + ], + }); + + const result = await runConsolidate( + { ...store, completeJson }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100 } + ); + + assert.equal(result.applied.length, 1, "the actionable subset must still merge"); + assert.equal(result.applied[0].survivorId, rows[0].id); + assert.deepEqual(result.applied[0].absorbedIds, [rows[2].id]); + assert.equal(store.rows.length, 3, "merge is non-destructive: all 3 rows must still be present"); + const absorbedRow = store.rows.find((r) => r.id === rows[2].id); + assert.ok(absorbedRow, "the absorbed preference duplicate must not be hard-deleted, only invalidated"); + assert.ok(JSON.parse(absorbedRow.metadata).invalidated_at, "the absorbed row must be marked invalidated"); + assert.ok( + store.rows.some((r) => r.id === rows[1].id), + "the unreferenced append-only events row must remain completely untouched" + ); + }); + + it("plugin-wide invariant: no LLM verdict path (merge or supersede) ever calls a hard delete", async () => { + // deps intentionally has NO delete method at all -- if either + // applyMergeVerdict or applySupersedeVerdict tried to call it, this + // would throw "deps.delete is not a function" and fail the test. + const ts = 1_700_000_000_000; + // Deliberately disjoint vocabulary between the two pairs (no shared + // words at all) so neither the reversal-gated nor the ratio-gated + // topic-overlap fallback can bridge them into one cluster -- this test + // is only about the delete-method invariant, not clustering. + const mergeRows = [ + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts + 1 }), + ]; + const supersedeRows = [ + makeRow({ abstract: "Desk setup: standing desk", factKey: "preferences:desk setup", vector: [0, 1, 0, 0], timestamp: ts + 2 }), + makeRow({ abstract: "Desk setup: no longer using a standing desk", factKey: "preferences:desk setup", vector: [0, 1, 0, 0], timestamp: ts + 3 }), + ]; + const rows = [...mergeRows, ...supersedeRows]; + const store = makeFakeStore(rows); + const { delete: _omittedDelete, ...storeWithoutDelete } = store; + + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }, + { cluster_index: 2, verdict: "supersede", survivor_index: 2, absorbed_indices: [1], reason: "reversal" }, + ], + }; + } + return { abstract: "merged", overview: "", content: "merged" }; + }; + + const result = await runConsolidate( + { ...storeWithoutDelete, completeJson }, + { scope: "global", apply: true, autoConfirm: true, now: ts + 100_000 } + ); + + assert.equal(result.applied.length, 2, "both verdicts must apply successfully without a delete method available"); + assert.equal(store.rows.length, 4, "no row may be removed by either verdict path"); + }); + + it("merge is idempotent: a second apply run over the same store makes zero further changes", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; + const store = makeFakeStore(rows); + const completeJson = async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }], + }; + } + return { abstract: "merged", overview: "", content: "merged" }; + }; + + const first = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: ts + 100_000 }); + const second = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: ts + 200_000 }); + + assert.equal(first.applied.length, 1); + assert.equal(second.applied.length, 0, "the invalidated absorbed row must not re-enter clustering on the next run"); + assert.equal(store.rows.length, 2, "still non-destructive: both rows remain present after two runs"); + }); + + it("decides multiple independent clusters with exactly ONE completeJson call, not one call per cluster", async () => { + const ts = 1_700_000_000_000; + const rows = [ + // Cluster A: coffee order duplicates + makeRow({ abstract: "Coffee order: oat milk latte", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts }), + makeRow({ abstract: "Coffee order: oat milk latte, extra hot", factKey: "preferences:coffee order", vector: [1, 0, 0, 0], timestamp: ts + 1 }), + // Cluster B: unrelated tea duplicates + makeRow({ abstract: "Tea order: chamomile", factKey: "preferences:tea order", vector: [0, 1, 0, 0], timestamp: ts + 2 }), + makeRow({ abstract: "Tea order: chamomile, no sugar", factKey: "preferences:tea order", vector: [0, 1, 0, 0], timestamp: ts + 3 }), + // Cluster C: unrelated desk duplicates + makeRow({ abstract: "Desk setup: standing desk", factKey: "preferences:desk setup", vector: [0, 0, 1, 0], timestamp: ts + 4 }), + makeRow({ abstract: "Desk setup: standing desk, oak top", factKey: "preferences:desk setup", vector: [0, 0, 1, 0], timestamp: ts + 5 }), + ]; + const store = makeFakeStore(rows); + let decideCallCount = 0; + const completeJson = async (_prompt, label) => { + if (label !== "consolidate-decide") { + return { abstract: "merged", overview: "", content: "merged" }; + } + decideCallCount += 1; + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "coffee dup" }, + { cluster_index: 2, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "tea dup" }, + { cluster_index: 3, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "desk dup" }, + ], + }; + }; + + const result = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: ts + 100_000 }); + + assert.equal(decideCallCount, 1, "all 3 clusters must be decided in a single completeJson call"); + assert.equal(result.clusters.length, 3, "all 3 clusters must be reported"); + assert.equal(result.applied.length, 3, "all 3 clusters' merge verdicts must be applied"); + }); + + it("never touches rows outside the requested scope", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ scope: "global", abstract: "Scoped fact one", factKey: "preferences:x", vector: [1, 0], timestamp: ts }), + makeRow({ scope: "other-scope", abstract: "Different scope fact", factKey: "preferences:x", vector: [1, 0], timestamp: ts }), + ]; + const store = makeFakeStore(rows); + const completeJson = async () => ({ + verdicts: [{ cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }], + }); + + const result = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: false, autoConfirm: true, now: ts + 100 }); + assert.equal(result.scanned, 1, "fetchRows must only see the requested scope"); + }); + + it("end-to-end: the decider sees the cross-lane favorite-drink family as ONE cluster, not fragmented or missed (item 4 acceptance)", async () => { + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ abstract: "Favorite drink: cola", content: "x", factKey: "preferences:favorite drink", source: "manual", vector: [1, 0, 0, 0], timestamp: ts }), + makeRow({ abstract: "The user really likes cola as their favorite drink", content: "y", factKey: undefined, source: "reflection", vector: [0, 1, 0, 0], timestamp: ts + 1 }), + makeRow({ abstract: "Cola is what gets ordered most evenings", content: "z", factKey: undefined, source: "auto-capture", vector: [0, 0, 1, 0], timestamp: ts + 2 }), + makeRow({ abstract: "No longer drinks cola", content: "w", factKey: undefined, source: "manual", vector: [0, 0, 0, 1], timestamp: ts + 3 }), + ]; + const store = makeFakeStore(rows); + let sawClusterMemberCount = null; + const completeJson = async (prompt, label) => { + if (label !== "consolidate-decide") return null; + sawClusterMemberCount = (prompt.match(/^\d+\. \[/gm) || []).length; + return { + verdicts: [ + { cluster_index: 1, verdict: "supersede", survivor_index: 4, absorbed_indices: [1, 2, 3], reason: "reversal supersedes all 3 cross-lane duplicates" }, + ], + }; + }; + + const result = await runConsolidate({ ...store, completeJson }, { scope: "global", apply: true, autoConfirm: true, now: ts + 100_000 }); + + assert.equal(result.clusters.length, 1, "exactly one cluster must reach the decider"); + assert.equal(sawClusterMemberCount, 4, "the decider's prompt must list all 4 rows together in that one cluster"); + assert.equal(result.applied.length, 1); + assert.equal(result.applied[0].absorbedIds.length, 3, "all 3 cross-lane duplicates must be absorbed by the single supersede verdict"); + }); +}); + +describe("memory consolidate: CLI attachment", () => { + it("registers consolidate as a subcommand of the memory-pro group, not the root program", () => { + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + + const program = new Command(); + const stubContext = { + store: {}, + retriever: {}, + scopeManager: {}, + migrator: {}, + }; + createMemoryCLI(stubContext)({ program }); + + const memoryPro = program.commands.find((c) => c.name() === "memory-pro"); + assert.ok(memoryPro, "memory-pro group command must be registered"); + + const groupNames = memoryPro.commands.map((c) => c.name()); + assert.ok( + groupNames.includes("consolidate"), + `expected "consolidate" under the memory-pro group, got: ${groupNames.join(", ")}` + ); + + const rootNames = program.commands.map((c) => c.name()); + assert.ok( + !rootNames.includes("consolidate"), + `"consolidate" must not be reachable as a root-level command, root has: ${rootNames.join(", ")}` + ); + }); +}); + +describe("memory consolidate: CLI system-prompt wiring", () => { + it("forwards buildConsolidatePrompt's system prompt through the CLI's completeJson adapter for both the decide and merge calls", async () => { + // Root cause (live-proven): the CLI's deps adapter used to be a 2-param + // lambda `(prompt, label) => llmClient.completeJson(prompt, label)`, so + // the decider's system prompt never reached the model -- every call ran + // under the generic "memory extraction assistant" system message and the + // model returned extraction-shaped JSON instead of a verdict. This test + // drives the real command action (not runConsolidate directly) so it + // actually exercises the adapter closure in cli.ts, not just consolidate.ts. + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + + const ts = 1_700_000_000_000; + const rows = [ + makeRow({ + scope: "agent:testbot", + abstract: "Coffee order: oat milk latte", + content: "User orders an oat milk latte.", + factKey: "preferences:coffee order", + vector: [1, 0], + timestamp: ts, + }), + makeRow({ + scope: "agent:testbot", + abstract: "Coffee order: oat milk latte, extra hot", + content: "User specified extra hot as well.", + factKey: "preferences:coffee order", + vector: [1, 0], + timestamp: ts + 1000, + }), + ]; + + const calls = []; + const context = { + store: { + fetchForCompaction: async (maxTimestamp, scopeFilter, limit) => + rows + .filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp) + .slice(0, limit ?? rows.length), + update: async () => ({}), + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + delete: async () => true, + }, + retriever: {}, + scopeManager: {}, + migrator: {}, + embedder: { embedPassage: async () => [1, 0] }, + llmClient: { + completeJson: async (prompt, label, system, temperature) => { + calls.push({ label, system, temperature }); + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "same fact, second row adds detail" }, + ], + }; + } + return { + results: [ + { index: 1, abstract: "Coffee order: oat milk latte, extra hot", overview: "", content: "merged content" }, + ], + }; + }, + getLastError: () => null, + }, + }; + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + await program.parseAsync(["node", "openclaw", "memory-pro", "consolidate", "--agent", "testbot", "--apply", "--yes"]); + + const decide = calls.find((c) => c.label === "consolidate-decide"); + assert.ok(decide, "expected a consolidate-decide completeJson call"); + assert.ok(decide.system, "the CLI adapter dropped the decider's system prompt"); + assert.match(decide.system, /consolidation decider/i); + assert.equal(decide.temperature, 0, "the CLI adapter dropped the decider's temperature override"); + + const merge = calls.find((c) => c.label === "consolidate-merge-batch"); + assert.ok(merge, "expected a consolidate-merge-batch completeJson call"); + assert.ok(merge.system, "the CLI adapter dropped the merge writer's system prompt"); + assert.match(merge.system, /merge writer/i); + }); +}); + +describe("memory-pro stats: live/total split (nit)", () => { + it("prints both live and total counts in the human-readable output, matching what --json already exposes", async () => { + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + + const context = { + store: { + stats: async () => ({ + totalCount: 5, + liveCount: 3, + scopeCounts: { global: 5 }, + categoryCounts: { preference: 5 }, + }), + hasFtsSupport: true, + }, + retriever: { getConfig: () => ({ mode: "hybrid" }) }, + scopeManager: { getStats: () => ({ totalScopes: 1 }) }, + migrator: {}, + }; + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(" ")); + try { + await program.parseAsync(["node", "openclaw", "memory-pro", "stats"]); + } finally { + console.log = originalLog; + } + + const output = logs.join("\n"); + assert.match(output, /Live memories:\s*3/i, "human-readable output must show the live count, like --json does"); + assert.match(output, /Total memories:\s*5/i); + }); +}); + +describe("memory consolidate: CLI journal-mirror agent identity", () => { + function buildContext(rows, mirrorCalls) { + return { + store: { + fetchForCompaction: async (maxTimestamp, scopeFilter, limit) => + rows + .filter((r) => (!scopeFilter || scopeFilter.includes(r.scope)) && r.timestamp <= maxTimestamp) + .slice(0, limit ?? rows.length), + update: async () => ({}), + getById: async (id) => { + const row = rows.find((r) => r.id === id); + return row ? { ...row } : null; + }, + delete: async () => true, + }, + retriever: {}, + scopeManager: {}, + migrator: {}, + embedder: { embedPassage: async () => [1, 0] }, + llmClient: { + completeJson: async (_prompt, label) => { + if (label === "consolidate-decide") { + return { + verdicts: [ + { cluster_index: 1, verdict: "merge", survivor_index: 1, absorbed_indices: [2], reason: "dup" }, + ], + }; + } + return { abstract: "merged", overview: "", content: "merged" }; + }, + getLastError: () => null, + }, + mdMirror: async (entry, meta) => { + mirrorCalls.push({ entry, meta }); + }, + }; + } + + function buildRows() { + const ts = 1_700_000_000_000; + return [ + makeRow({ scope: "agent:agent-one", abstract: "Coffee order: oat milk latte", content: "a", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts }), + makeRow({ scope: "agent:agent-one", abstract: "Coffee order: oat milk latte, extra hot", content: "b", factKey: "preferences:coffee order", vector: [1, 0], timestamp: ts + 1000 }), + ]; + } + + it("threads --agent through to the journal-mirror writer's meta.agentId", async () => { + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + const mirrorCalls = []; + const context = buildContext(buildRows(), mirrorCalls); + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + await program.parseAsync([ + "node", "openclaw", "memory-pro", "consolidate", + "--apply", "--agent", "agent-one", "--yes", + ]); + + assert.equal(mirrorCalls.length, 1, "expected exactly one journal-mirror write for the applied merge"); + assert.equal(mirrorCalls[0].meta.agentId, "agent-one", "the CLI's --agent value must reach the journal writer"); + assert.match(mirrorCalls[0].meta.source, /memory-consolidate/); + }); + + it("refuses to run without --agent (scope is always derived as agent:)", async () => { + const { createMemoryCLI } = jiti(path.join(testDir, "..", "cli.ts")); + const mirrorCalls = []; + const context = buildContext(buildRows(), mirrorCalls); + + const program = new Command(); + program.exitOverride(); + createMemoryCLI(context)({ program }); + + await assert.rejects( + () => program.parseAsync(["node", "openclaw", "memory-pro", "consolidate", "--apply", "--yes"]), + (err) => err instanceof Error && err.code === "commander.missingMandatoryOptionValue", + ); + assert.equal(mirrorCalls.length, 0, "nothing may run or write without an agent identity"); + }); +}); diff --git a/test/migrate-legacy-schema.test.mjs b/test/migrate-legacy-schema.test.mjs index 34ae70096..ca23c27b0 100644 --- a/test/migrate-legacy-schema.test.mjs +++ b/test/migrate-legacy-schema.test.mjs @@ -74,13 +74,19 @@ describe("legacy LanceDB migration", () => { }); it("skips re-import when skipExisting is enabled and the legacy id already exists", async () => { + // Legacy-epoch-seconds-style values (normalizeMemoryTimestamp treats + // anything under LEGACY_SECONDS_TIMESTAMP_MAX as seconds and multiplies + // by 1000). 1700000000 seconds -> Nov 2023, safely in the past -- + // 2222222222 seconds would normalize to ~2040, a FUTURE timestamp that + // store.list()'s excludeInactive default (item 6) correctly treats as + // not-yet-active and excludes from the read-back assertion below. const legacyPath = await createLegacyDb([ { id: "legacy-keep-id", text: "keep the original identifier", importance: 0.6, category: "decision", - createdAt: 2222222222, + createdAt: 1700000000, vector: [1, 0, 0, 0], scope: "agent:main", }, @@ -94,7 +100,7 @@ describe("legacy LanceDB migration", () => { category: "decision", scope: "agent:main", importance: 0.6, - timestamp: 2222222222, + timestamp: 1700000000, metadata: "{}", }); diff --git a/test/store-empty-scope-filter.test.mjs b/test/store-empty-scope-filter.test.mjs index 73a78393e..cf07d224c 100644 --- a/test/store-empty-scope-filter.test.mjs +++ b/test/store-empty-scope-filter.test.mjs @@ -32,6 +32,7 @@ describe("MemoryStore empty scopeFilter semantics", () => { assert.deepStrictEqual(await store.bm25Search("test", 5, []), []); assert.deepStrictEqual(await store.stats([]), { totalCount: 0, + liveCount: 0, scopeCounts: {}, categoryCounts: {}, }); diff --git a/test/store-excludeinactive-default.test.mjs b/test/store-excludeinactive-default.test.mjs new file mode 100644 index 000000000..2ae501ac3 --- /dev/null +++ b/test/store-excludeinactive-default.test.mjs @@ -0,0 +1,182 @@ +// test/store-excludeinactive-default.test.mjs +// +// Item 6 (PR #946 fix round): invalidated/superseded rows must be invisible +// to read surfaces BY DEFAULT, not opt-in. This exercises the store-layer +// choke point directly against a real temporary LanceDB instance (not a +// fake/mocked table) -- store.ts's own typed-array vector bug shipped +// invisibly through mocked-vector unit tests, so any store.ts read-path +// change in this codebase gets a real round-trip test. + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +function makeStore(prefix, vectorDim = 4) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + const { MemoryStore } = jiti("../src/store.ts"); + return { store: new MemoryStore({ dbPath: dir, vectorDim }), dir }; +} + +async function storeLiveAndInvalidatedPair(store) { + const live = await store.store({ + text: "Live fact: user likes cola", + vector: [1, 0, 0, 0], + category: "preference", + scope: "test", + importance: 0.7, + metadata: JSON.stringify({ + l0_abstract: "Live fact: user likes cola", + memory_category: "preferences", + valid_from: Date.now() - 10_000, + }), + }); + + const dead = await store.store({ + text: "Dead fact: user liked tea (superseded)", + vector: [1, 0, 0, 0], + category: "preference", + scope: "test", + importance: 0.7, + metadata: JSON.stringify({ + l0_abstract: "Dead fact: user liked tea (superseded)", + memory_category: "preferences", + valid_from: Date.now() - 20_000, + }), + }); + + // Invalidate the second row the same way supersede/consolidate does: + // stamp invalidated_at into the metadata blob via a normal update(). + await store.update(dead.id, { + metadata: JSON.stringify({ + l0_abstract: "Dead fact: user liked tea (superseded)", + memory_category: "preferences", + valid_from: Date.now() - 20_000, + invalidated_at: Date.now() - 5_000, + superseded_by: live.id, + }), + }); + + return { live, dead }; +} + +describe("store.ts: excludeInactive defaults to true (item 6 choke point)", () => { + it("vectorSearch excludes the invalidated row by default", async () => { + const { store, dir } = makeStore("excl-vs-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.vectorSearch([1, 0, 0, 0], 10, 0, ["test"]); + const ids = results.map((r) => r.entry.id); + assert.ok(ids.includes(live.id), "the live row must be returned"); + assert.ok(!ids.includes(dead.id), "the invalidated row must NOT be returned by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("vectorSearch still returns the invalidated row when excludeInactive:false is explicit", async () => { + const { store, dir } = makeStore("excl-vs-optout-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.vectorSearch([1, 0, 0, 0], 10, 0, ["test"], { excludeInactive: false }); + const ids = results.map((r) => r.entry.id); + assert.ok(ids.includes(live.id)); + assert.ok(ids.includes(dead.id), "explicit opt-out must still surface the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("bm25Search excludes the invalidated row by default", async () => { + const { store, dir } = makeStore("excl-bm-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.bm25Search("fact", 10, ["test"]); + const ids = results.map((r) => r.entry.id); + assert.ok(ids.includes(live.id)); + assert.ok(!ids.includes(dead.id), "the invalidated row must NOT be returned by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("list() excludes the invalidated row by default", async () => { + const { store, dir } = makeStore("excl-list-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.list(["test"], undefined, 20, 0); + const ids = results.map((r) => r.id); + assert.ok(ids.includes(live.id)); + assert.ok(!ids.includes(dead.id), "list() must exclude invalidated rows by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("list() surfaces the invalidated row when excludeInactive:false is explicit", async () => { + const { store, dir } = makeStore("excl-list-optout-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.list(["test"], undefined, 20, 0, { excludeInactive: false }); + const ids = results.map((r) => r.id); + assert.ok(ids.includes(live.id)); + assert.ok(ids.includes(dead.id), "explicit opt-out must still surface the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fetchForCompaction() excludes the invalidated row by default", async () => { + const { store, dir } = makeStore("excl-fc-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.fetchForCompaction(Date.now() + 1000, ["test"], 200); + const ids = results.map((r) => r.id); + assert.ok(ids.includes(live.id)); + assert.ok(!ids.includes(dead.id), "fetchForCompaction() must exclude invalidated rows by default"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fetchForCompaction() surfaces the invalidated row when excludeInactive:false is explicit", async () => { + const { store, dir } = makeStore("excl-fc-optout-"); + try { + const { live, dead } = await storeLiveAndInvalidatedPair(store); + const results = await store.fetchForCompaction(Date.now() + 1000, ["test"], 200, { excludeInactive: false }); + const ids = results.map((r) => r.id); + assert.ok(ids.includes(live.id)); + assert.ok(ids.includes(dead.id), "explicit opt-out must still surface the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("getById stays unfiltered by design -- it must still return an invalidated row", async () => { + const { store, dir } = makeStore("excl-getbyid-"); + try { + const { dead } = await storeLiveAndInvalidatedPair(store); + const found = await store.getById(dead.id, ["test"]); + assert.ok(found, "getById must never filter by invalidation status"); + assert.equal(found.id, dead.id); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("stats() reports both a blended totalCount and a live-only liveCount", async () => { + const { store, dir } = makeStore("excl-stats-"); + try { + await storeLiveAndInvalidatedPair(store); + const stats = await store.stats(["test"]); + assert.equal(stats.totalCount, 2, "totalCount must still include the invalidated row"); + assert.equal(stats.liveCount, 1, "liveCount must exclude the invalidated row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/store-list-stats-projection-fallback.test.mjs b/test/store-list-stats-projection-fallback.test.mjs index 6fc0fb51f..e6eb664f3 100644 --- a/test/store-list-stats-projection-fallback.test.mjs +++ b/test/store-list-stats-projection-fallback.test.mjs @@ -68,6 +68,7 @@ describe("MemoryStore list/stats projection fallback", () => { assert.deepEqual(await store.stats(), { totalCount: 1, + liveCount: 1, scopeCounts: { global: 1 }, categoryCounts: { fact: 1 }, }); @@ -89,6 +90,7 @@ describe("MemoryStore list/stats projection fallback", () => { assert.deepEqual(await store.stats(), { totalCount: 0, + liveCount: 0, scopeCounts: {}, categoryCounts: {}, }); diff --git a/test/temporal-facts.test.mjs b/test/temporal-facts.test.mjs index 970a599d4..a28a97666 100644 --- a/test/temporal-facts.test.mjs +++ b/test/temporal-facts.test.mjs @@ -174,7 +174,10 @@ async function runTest() { assert.equal(stats.created, 1); assert.equal(stats.superseded, 1); - const entries = await store.list(["test"], undefined, 10, 0); + // excludeInactive:false: this test explicitly verifies the invalidated + // historical row is retained (not hard-deleted), so it needs the + // full/forensic view, not the excludeInactive-by-default read (item 6). + const entries = await store.list(["test"], undefined, 10, 0, { excludeInactive: false }); assert.equal(entries.length, 2, "supersede should keep old + new entries"); const currentEntry = entries.find((entry) => entry.text.includes("咖啡")); @@ -255,7 +258,9 @@ async function runTest() { } // Verify there are now 10 total entries (1 original + 1 current + 8 history) - const allEntries = await store.list(["test"], undefined, 20, 0); + // excludeInactive:false: most of these 10 rows are invalidated_at-stamped + // history, so this forensic count needs the full view (item 6). + const allEntries = await store.list(["test"], undefined, 20, 0, { excludeInactive: false }); assert.equal(allEntries.length, 10, "should have 10 entries total"); const crowdedResults = await retriever.retrieve({