feat: add a captureAssistant context mode for non-extractable assistant context

dist/index.js

@@ -39,7 +39,7 @@ import { createReflectionEventId } from "./src/reflection-event-store.js";
 import { buildReflectionMappedMetadata } from "./src/reflection-mapped-metadata.js";
 import { createMemoryCLI } from "./cli.js";
 import { isNoise } from "./src/noise-filter.js";
-import { normalizeAutoCaptureText } from "./src/auto-capture-cleanup.js";
+import { normalizeAutoCaptureText, buildConversationTurnsForExtraction, trimTurnsToUserCap, dedupePairWindow, } from "./src/auto-capture-cleanup.js";
 // Import smart extraction & lifecycle components
 import { SmartExtractor, createExtractionRateLimiter } from "./src/smart-extractor.js";
 import { compressTexts, estimateConversationValue } from "./src/session-compressor.js";
@@ -1845,6 +1845,7 @@ function _initPluginState(api) {
             noiseBank.init(embedder).catch((err) => api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`));
             const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter(config, resolvedDbPath, api);
             smartExtractor = new SmartExtractor(store, embedder, llmClient, {
+                captureAssistantEligible: config.captureAssistant === true,
                 user: "User",
                 extractMinMessages: config.extractMinMessages ?? 4,
                 extractMaxChars: config.extractMaxChars ?? 8000,
@@ -1885,6 +1886,7 @@ function _initPluginState(api) {
     const autoCaptureSeenTextCount = new Map();
     const autoCapturePendingIngressTexts = new Map();
     const autoCaptureRecentTexts = new Map();
+    const autoCaptureRecentPairTurns = new Map();
     return {
         config,
         resolvedDbPath,
@@ -1912,6 +1914,7 @@ function _initPluginState(api) {
         autoCaptureSeenTextCount,
         autoCapturePendingIngressTexts,
         autoCaptureRecentTexts,
+        autoCaptureRecentPairTurns,
     };
 }
 export function isAgentOrSessionExcluded(agentId, sessionKey, patterns) {
@@ -2020,7 +2023,7 @@ const memoryLanceDBProPlugin = {
             _registeredApisMap.delete(api); // dual-track rollback: Map un-claim
             throw err;
         }
-        const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, } = singleton;
+        const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, autoCaptureRecentPairTurns, } = singleton;
         warnForDisabledChannelPlugin(api.config, api.logger);
         async function sleep(ms, signal) {
             if (signal?.aborted) {
@@ -2900,29 +2903,36 @@ const memoryLanceDBProPlugin = {
                             ? config.scopes?.default ?? "global"
                             : scopeManager.getDefaultScope(agentId);
                         const sessionKey = ctx?.sessionKey || event.sessionKey || "unknown";
-                        api.logger.debug(`memory-lancedb-pro: auto-capture agent_end payload for agent ${agentId} (sessionKey=${sessionKey}, captureAssistant=${config.captureAssistant === true}, ${summarizeAgentEndMessages(event.messages)})`);
+                        api.logger.debug(`memory-lancedb-pro: auto-capture agent_end payload for agent ${agentId} (sessionKey=${sessionKey}, captureAssistant=${JSON.stringify(config.captureAssistant)}, ${summarizeAgentEndMessages(event.messages)})`);
                         // Extract text content from messages
                         const eligibleTexts = [];
+                        const assistantContextTexts = [];
+                        const conversationTurns = [];
                         let skippedAutoCaptureTexts = 0;
+                        const captureAssistantValue = config.captureAssistant;
+                        const captureAssistantEligible = captureAssistantValue === true;
+                        const captureAssistantAsContext = captureAssistantValue === "context";
                         for (const msg of event.messages) {
                             if (!msg || typeof msg !== "object") {
                                 continue;
                             }
                             const msgObj = msg;
                             const role = msgObj.role;
-                            const captureAssistant = config.captureAssistant === true;
-                            if (role !== "user" &&
-                                !(captureAssistant && role === "assistant")) {
+                            const isEligibleRole = role === "user" || (captureAssistantEligible && role === "assistant");
+                            const isContextOnlyRole = captureAssistantAsContext && role === "assistant";
+                            if (!isEligibleRole && !isContextOnlyRole) {
                                 continue;
                             }
+                            const targetTexts = isEligibleRole ? eligibleTexts : assistantContextTexts;
                             const content = msgObj.content;
                             if (typeof content === "string") {
                                 const normalized = normalizeAutoCaptureText(role, content, shouldSkipReflectionMessage);
                                 if (!normalized) {
                                     skippedAutoCaptureTexts++;
                                 }
                                 else {
-                                    eligibleTexts.push(normalized);
+                                    targetTexts.push(normalized);
+                                    conversationTurns.push({ role: role, text: normalized });
                                 }
                                 continue;
                             }
@@ -2940,7 +2950,8 @@ const memoryLanceDBProPlugin = {
                                             skippedAutoCaptureTexts++;
                                         }
                                         else {
-                                            eligibleTexts.push(normalized);
+                                            targetTexts.push(normalized);
+                                            conversationTurns.push({ role: role, text: normalized });
                                         }
                                     }
                                 }
@@ -2978,6 +2989,24 @@ const memoryLanceDBProPlugin = {
                             pruneMapIfOver(autoCaptureRecentTexts, AUTO_CAPTURE_MAP_MAX_ENTRIES);
                         }
                         const minMessages = config.extractMinMessages ?? 4;
+                        // Rolling PAIR window (operator spec: extractMinMessages counts
+                        // user<->assistant pairs, and captureAssistant context rides the
+                        // same window). This call's new pairs -- kept user turns with the
+                        // assistant replies interleaved in true order -- extend what
+                        // earlier calls buffered, bounded to extractMinMessages user turns
+                        // (or this call's own new-user count when larger, so unextracted
+                        // user turns are never trimmed out of their own transcript).
+                        const thisCallPairTurns = buildConversationTurnsForExtraction({
+                            messageLoopTurns: conversationTurns,
+                            eligibleTexts,
+                            newUserTexts: newTexts,
+                        });
+                        const priorPairTurns = autoCaptureRecentPairTurns.get(sessionKey) || [];
+                        const pairWindowTurns = trimTurnsToUserCap(dedupePairWindow([...priorPairTurns, ...thisCallPairTurns]), Math.max(minMessages, thisCallPairTurns.filter((turn) => turn.role === "user").length));
+                        if (thisCallPairTurns.length > 0) {
+                            autoCaptureRecentPairTurns.set(sessionKey, pairWindowTurns);
+                            pruneMapIfOver(autoCaptureRecentPairTurns, AUTO_CAPTURE_MAP_MAX_ENTRIES);
+                        }
                         if (skippedAutoCaptureTexts > 0) {
                             api.logger.debug(`memory-lancedb-pro: auto-capture skipped ${skippedAutoCaptureTexts} injected/system text block(s) for agent ${agentId}`);
                         }
@@ -3033,10 +3062,17 @@ const memoryLanceDBProPlugin = {
                             if (cumulativeCount >= minMessages) {
                                 api.logger.debug(`memory-lancedb-pro: auto-capture running smart extraction for agent ${agentId} (cumulative=${cumulativeCount} >= minMessages=${minMessages}, cleanTexts=${cleanTexts.length})`);
                                 const conversationText = cleanTexts.join("\n");
+                                // The pair window is the transcript; user turns the noise
+                                // filter dropped stay out of it so they cannot become sources.
+                                const noiseDroppedTexts = new Set(texts.filter((text) => !cleanTexts.includes(text)));
+                                const finalConversationTurns = pairWindowTurns.filter((turn) => !(turn.role === "user" && noiseDroppedTexts.has(turn.text)));
+                                const assistantWindowTexts = finalConversationTurns
+                                    .filter((turn) => turn.role === "assistant")
+                                    .map((turn) => turn.text);
                                 // issue #417 Fix #10: prevent hook crash on LLM API errors / network timeouts
                                 let stats = null;
                                 try {
-                                    stats = await smartExtractor.extractAndPersist(conversationText, sessionKey, { scope: defaultScope, scopeFilter: accessibleScopes, agentId });
+                                    stats = await smartExtractor.extractAndPersist(conversationText, sessionKey, { scope: defaultScope, scopeFilter: accessibleScopes, agentId, assistantContextTexts: assistantWindowTexts, conversationTurns: finalConversationTurns });
                                 }
                                 catch (err) {
                                     api.logger.error(`memory-lancedb-pro: smart-extract failed for agent ${agentId}: ${String(err)}`);
@@ -3056,6 +3092,7 @@ const memoryLanceDBProPlugin = {
                                     // consumed history length there instead, so the next turn
                                     // only sees the delta.
                                     autoCaptureSeenTextCount.set(sessionKey, pendingIngressTexts.length > 0 ? 0 : eligibleTexts.length);
+                                    autoCaptureRecentPairTurns.delete(sessionKey);
                                     return; // Smart extraction handled everything
                                 }
                                 if ((stats.boundarySkipped ?? 0) === 0) {
@@ -4066,10 +4103,15 @@ const memoryLanceDBProPlugin = {
                 const now = new Date(params.timestampMs ?? Date.now());
                 const dateStr = now.toISOString().split("T")[0];
                 const timeStr = now.toISOString().split("T")[1].split(".")[0];
+                // Session key/id stay out of `text`: it is the FTS index surface, and
+                // the `simple` tokenizer splits a key like
+                // `agent:main:cron:<uuid>:run:<uuid>` on its punctuation — so every session
+                // summary ends up indexed under `agent`, `main`, `cron`, `run`. A query
+                // mentioning any of those then BM25-matches every session summary in the
+                // store regardless of content. Both ids are already recorded structurally
+                // in metadata below, so provenance is unaffected.
                 const memoryText = [
                     `Session: ${dateStr} ${timeStr} UTC`,
-                    `Session Key: ${params.sessionKey}`,
-                    `Session ID: ${params.sessionId}`,
                     `Source: ${params.source}`,
                     "",
                     "Conversation Summary:",
@@ -4685,7 +4727,7 @@ export function parsePluginConfig(value) {
             }
             return s;
         })(),
-        captureAssistant: cfg.captureAssistant === true,
+        captureAssistant: cfg.captureAssistant === "context" ? "context" : cfg.captureAssistant === true,
         retrieval: typeof cfg.retrieval === "object" && cfg.retrieval !== null
             ? (() => {
                 const retrieval = { ...cfg.retrieval };

dist/src/auto-capture-cleanup.js

@@ -118,3 +118,149 @@ export function normalizeAutoCaptureText(role, text, shouldSkipMessage) {
         return null;
     return normalized;
 }
+/**
+ * Renders turns oldest-first as a continuous "Label: text" transcript, one
+ * line per turn, no blank lines or per-turn metadata between them. `userLabel`
+ * replaces the generic "User" label when a configured name is known;
+ * assistant turns always render as "Assistant" (no per-agent name surface).
+ */
+export function formatConversationTranscript(turns, userLabel = "User") {
+    return turns
+        .map((turn) => `${turn.role === "user" ? userLabel : "Assistant"}: ${turn.text}`)
+        .join("\n");
+}
+/**
+ * Assembles the ordered turn sequence for the extraction prompt's transcript
+ * from this call's true message-loop order, without recomputing any
+ * eligibility or watermark decision -- it only consumes their already-decided
+ * results. The window is PAIR-shaped: an assistant turn belongs to the user
+ * turn it follows (a reply), so dropping an already-extracted user turn also
+ * drops the assistant turns of its pair.
+ * - `newUserTexts` narrower than `eligibleTexts` (watermark tail-slice):
+ *   drop that many leading user turns AND every assistant turn that precedes
+ *   the first kept user turn (they are replies to the dropped pairs).
+ * - `newUserTexts` not a tail-slice of `eligibleTexts` at all (pending-ingress
+ *   replay from a different source, no per-message role correlation
+ *   available): fall back to flat user turns for the replayed content.
+ * Prior-call assistant context is never resurrected here -- pair windows
+ * carry across calls through the caller's rolling pair buffer (see
+ * trimTurnsToUserCap), not through a separate assistant-only carry.
+ */
+export function buildConversationTurnsForExtraction(params) {
+    const { messageLoopTurns, eligibleTexts, newUserTexts } = params;
+    const isTailSliceOfEligible = newUserTexts.length <= eligibleTexts.length &&
+        eligibleTexts
+            .slice(eligibleTexts.length - newUserTexts.length)
+            .every((text, i) => text === newUserTexts[i]);
+    if (!isTailSliceOfEligible) {
+        return newUserTexts.map((text) => ({ role: "user", text }));
+    }
+    const skipUserCount = eligibleTexts.length - newUserTexts.length;
+    const thisCallTurns = [];
+    let userSeen = 0;
+    for (const turn of messageLoopTurns) {
+        if (turn.role === "user") {
+            userSeen++;
+            if (userSeen <= skipUserCount)
+                continue;
+        }
+        else if (userSeen <= skipUserCount) {
+            // Reply to a dropped (already-extracted) user turn: goes with its pair.
+            continue;
+        }
+        thisCallTurns.push(turn);
+    }
+    return thisCallTurns;
+}
+/**
+ * Bounds a rolling pair window to at most `maxUserTurns` user turns, keeping
+ * the newest ones with their interleaved assistant replies, and never leaving
+ * an orphan assistant turn ahead of the window's first user turn. This is how
+ * extractMinMessages acts as the window size in PAIRS: the caller passes
+ * max(extractMinMessages, this call's new user turns), so the transcript
+ * always contains every not-yet-extracted user turn, padded with earlier
+ * still-buffered pairs up to the configured window.
+ */
+export function trimTurnsToUserCap(turns, maxUserTurns) {
+    const cap = Math.max(1, maxUserTurns);
+    let userCount = 0;
+    let start = turns.length;
+    for (let i = turns.length - 1; i >= 0; i--) {
+        if (turns[i].role === "user") {
+            userCount++;
+            if (userCount > cap)
+                break;
+            start = i;
+        }
+    }
+    return turns.slice(start);
+}
+/**
+ * Repairs a pair window that double-preserved deferred turns. A below-threshold
+ * deferral keeps content alive on two independent paths -- the rolling pair
+ * buffer, and the watermark rollback (or pending-ingress re-queue) whose next
+ * slice re-includes the same turns -- so the assembled window can carry the
+ * same exchange twice. Collapse duplicates by user text at pair granularity:
+ * a pair-shaped copy (user turn plus its replies) beats a flat re-queued copy,
+ * copies of an identical exchange collapse to the latest, and a repeated user
+ * text whose replies differ is a real conversation and is kept whole.
+ */
+export function dedupePairWindow(turns) {
+    const groups = [];
+    let current = null;
+    for (const turn of turns) {
+        if (turn.role === "user") {
+            current = { turns: [turn], userText: turn.text, replies: "" };
+            groups.push(current);
+        }
+        else if (current) {
+            current.turns.push(turn);
+            current.replies = JSON.stringify(current.turns.slice(1).map((t) => t.text));
+        }
+        else {
+            groups.push({ turns: [turn], userText: null, replies: "" });
+        }
+    }
+    const kept = [];
+    for (const group of groups) {
+        if (group.userText === null) {
+            kept.push(group);
+            continue;
+        }
+        let prevIndex = -1;
+        for (let i = kept.length - 1; i >= 0; i--) {
+            if (kept[i].userText === group.userText) {
+                prevIndex = i;
+                break;
+            }
+        }
+        if (prevIndex < 0) {
+            kept.push(group);
+            continue;
+        }
+        const prev = kept[prevIndex];
+        const prevPaired = prev.turns.length > 1;
+        const currPaired = group.turns.length > 1;
+        if (currPaired && prevPaired) {
+            if (prev.replies === group.replies) {
+                kept.splice(prevIndex, 1);
+                kept.push(group);
+            }
+            else {
+                kept.push(group);
+            }
+        }
+        else if (currPaired && !prevPaired) {
+            kept.splice(prevIndex, 1);
+            kept.push(group);
+        }
+        else if (!currPaired && prevPaired) {
+            continue;
+        }
+        else {
+            kept.splice(prevIndex, 1);
+            kept.push(group);
+        }
+    }
+    return kept.flatMap((group) => group.turns);
+}