fix(transfer): preserve Knowledge Pages during bank transfer (#3308)

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -4419,8 +4419,9 @@ async def import_bank_async(
         """Restore a whole bank from an :func:`transfer.export_bank` archive.
 
         Re-embeds facts with this instance's embedding model and rebuilds links and
-        indexes; restores bank config, mental models, directives and webhooks as
-        exported (no consolidation/webhooks — a migration restores exact state). The
+        indexes; restores bank config, mental model logical fields, directives and
+        webhooks as exported while rebuilding target-derived mental model search
+        state (no consolidation/webhooks — a migration restores exact state). The
         target bank must not already exist (import restores a whole bank, not a merge).
         """
         from .transfer import import_bank

hindsight-api-slim/hindsight_api/engine/transfer/export.py

@@ -70,7 +70,7 @@
 )
 # Carried verbatim as JSON rows (bank config + synthesized state). Embedding-bearing
 # rows have their vector stripped (see _DERIVED_COLUMNS) and are re-embedded on import.
-_BANK_ROW_TABLES = ("banks", "mental_models", "directives", "webhooks")
+_BANK_ROW_TABLES = ("banks", "mental_models", "knowledge_pages", "directives", "webhooks")
 # Bank-scoped child-history carried verbatim. Unlike observations, mental models
 # keep their (id, bank_id) across export/import, so their refresh history can be
 # re-attached. The surrogate ``id`` is dropped on dump so the target reassigns it
@@ -88,12 +88,6 @@
         # to fresh ids, so carrying them would only produce dangling associations.
         # Revert anything worth keeping on the source before migrating.
         "invalidated_memory_units",
-        # Knowledge-base folder/page tree (client-managed metadata over the carried
-        # mental models). Not carried yet: its self-referential parent_id FK needs a
-        # parents-first (topological) restore order, which the generic per-row
-        # _restore_rows doesn't provide — a follow-up. The mental models themselves
-        # ARE carried, so the target can recreate the tree.
-        "knowledge_pages",
     }
 )
 # Derived columns dropped from carried rows so the target regenerates them with
@@ -294,11 +288,12 @@ async def export_bank(
 
     Produces a superset of the documents archive: the logical
     document/fact/observation export (replayed and re-embedded on import) plus
-    the bank's config, mental models, directives and webhooks as JSON rows. With
-    ``include_history`` the operational tails (audit_log, llm_requests) are also
-    carried. Intended for migrating a bank to a new instance configured with a
-    different embedding model / vector / text-search backend — every vector is
-    regenerated on the target, so nothing here is encoder-specific.
+    the bank's config, mental models, knowledge pages, directives and webhooks as
+    JSON rows. With ``include_history`` the operational tails (audit_log,
+    llm_requests) are also carried. Intended for migrating a bank to a new
+    instance configured with a different embedding model / vector / text-search
+    backend — every vector is regenerated on the target, so nothing here is
+    encoder-specific.
 
     ``conn`` is a live connection scoped to the bank's schema (the admin CLI sets
     ``_current_schema`` and passes its raw connection; the engine acquires one

hindsight-api-slim/hindsight_api/engine/transfer/importer.py

@@ -21,7 +21,15 @@
 
 from ..causal_links import CANONICAL_CAUSAL_LINK_TYPE, LEGACY_CAUSAL_LINK_TYPES
 from ..db_utils import acquire_with_retry
-from ..retain import bank_utils, chunk_storage, embedding_processing, fact_storage, link_utils, orchestrator
+from ..retain import (
+    bank_utils,
+    chunk_storage,
+    embedding_processing,
+    embedding_utils,
+    fact_storage,
+    link_utils,
+    orchestrator,
+)
 from ..retain.types import (
     CausalRelation,
     ChunkMetadata,
@@ -230,7 +238,7 @@ async def import_documents(
 
 # Bank-level config/state tables restored verbatim from a whole-bank archive.
 # Order matters for foreign keys: banks (parent) is restored before any child.
-_BANK_CHILD_TABLES = ("mental_models", "directives", "webhooks")
+_BANK_CHILD_TABLES = ("mental_models", "knowledge_pages", "directives", "webhooks")
 # Child-history carried verbatim; restored after its parent (mental_models) so the
 # foreign key resolves. Surrogate ids were dropped on export (the target reassigns
 # them), so these restore via fresh IDENTITY values.
@@ -256,12 +264,31 @@ class ParsedBankArchive:
     """The bank-level sections of a whole-bank archive (documents read separately)."""
 
     manifest: TransferManifest
-    # table name -> list of verbatim row dicts (banks, mental_models, directives, webhooks)
+    # table name -> list of verbatim row dicts (banks, mental_models, knowledge_pages, directives, webhooks)
     bank_rows: dict[str, list[dict]] = field(default_factory=dict)
     # table name -> rows (audit_log, llm_requests), present only with --include-history
     history_rows: dict[str, list[dict]] = field(default_factory=dict)
 
 
+def _order_knowledge_pages(rows: list[dict]) -> list[dict]:
+    """Order knowledge nodes so every self-referential parent is inserted first."""
+    pending = list(rows)
+    ordered: list[dict] = []
+    restored_ids: set[str] = set()
+    while pending:
+        next_pending: list[dict] = []
+        for row in pending:
+            if row["parent_id"] is None or row["parent_id"] in restored_ids:
+                ordered.append(row)
+                restored_ids.add(row["id"])
+            else:
+                next_pending.append(row)
+        if len(next_pending) == len(pending):
+            raise ValueError("Knowledge-page archive contains an unresolved parent or cycle")
+        pending = next_pending
+    return ordered
+
+
 def parse_bank_archive(archive_bytes: bytes) -> ParsedBankArchive:
     """Parse the bank-level sections of a whole-bank archive (``archive_type='bank'``)."""
     with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as zf:
@@ -349,6 +376,67 @@ async def _restore_rows(
     return inserted
 
 
+async def _rebuild_mental_model_search_state(
+    conn: Any,
+    rows: list[dict],
+    *,
+    bank_id: str,
+    embedding_values: list[str | None],
+    config: Any,
+) -> None:
+    """Rebuild target-derived search fields for carried mental models.
+
+    Whole-bank archives deliberately omit ``embedding`` and ``search_vector``.
+    Mental models are restored as rows rather than through ``create_mental_model``,
+    so the target-side embedding and text-search lifecycle must be applied here.
+    Native generated search columns are populated by the INSERT; regular vector
+    columns (notably VectorChord, and native columns after the configurable-language
+    migration) need an explicit expression using the target configuration.
+    """
+    if not rows:
+        return
+
+    search_vector_expr: str | None = None
+    if config.database_backend == "postgresql":
+        from ..memory_engine import get_current_schema
+
+        schema = get_current_schema()
+        column = await conn.fetchrow(
+            """
+            SELECT is_generated
+            FROM information_schema.columns
+            WHERE table_schema = $1 AND table_name = 'mental_models' AND column_name = 'search_vector'
+            """,
+            schema,
+        )
+        if not (column and column["is_generated"] == "ALWAYS"):
+            from ..db.ops_postgresql import pg_search_vector_expr
+
+            search_vector_expr = pg_search_vector_expr(
+                config,
+                text_col="name",
+                context_col="''",
+                signals_col="content",
+            )
+
+    for row, embedding in zip(rows, embedding_values):
+        if search_vector_expr is None:
+            await conn.execute(
+                f"UPDATE {fq_table('mental_models')} SET embedding = $1 WHERE bank_id = $2 AND id = $3",
+                embedding,
+                bank_id,
+                row["id"],
+            )
+        else:
+            await conn.execute(
+                f"UPDATE {fq_table('mental_models')} SET embedding = $1, search_vector = {search_vector_expr} "
+                "WHERE bank_id = $2 AND id = $3",
+                embedding,
+                bank_id,
+                row["id"],
+            )
+
+
 async def import_bank(
     *,
     backend: Any,
@@ -372,7 +460,8 @@ async def import_bank(
     target id is present, this raises — delete it first or pass ``target_bank_id``
     for a fresh id. A migration restores *exact* state, so unlike the document
     import it fires no retain webhooks and triggers no consolidation/graph
-    maintenance: observations and mental models are restored as exported.
+    maintenance: observations and mental model logical fields are restored as
+    exported, while target-derived mental model search state is rebuilt.
 
     Takes ``resolve_config`` rather than a resolved config because the only correct
     moment to resolve one is *inside* this function, after the archive's bank row
@@ -451,11 +540,37 @@ async def import_bank(
         facts_imported=doc_result.facts_imported,
         observations_imported=doc_result.observations_imported,
     )
+    mental_model_rows = parsed.bank_rows.get("mental_models", [])
+    # A slow embedder must not pin a pooled connection while the target-side
+    # derived search fields are rebuilt below.
+    embeddings = (
+        await embedding_utils.generate_embeddings_batch(
+            embeddings_model,
+            [f"{row['name']} {row['content']}" for row in mental_model_rows],
+        )
+        if mental_model_rows
+        else []
+    )
+    mental_model_embedding_values = [str(value) if value else None for value in embeddings]
+
     async with acquire_with_retry(backend) as conn:
         result.mental_models_imported = await _restore_rows(
             conn,
             "mental_models",
-            parsed.bank_rows.get("mental_models", []),
+            mental_model_rows,
+            bank_rows_json_encoding=bank_rows_json_encoding,
+        )
+        await _rebuild_mental_model_search_state(
+            conn,
+            mental_model_rows,
+            bank_id=bank_id,
+            embedding_values=mental_model_embedding_values,
+            config=config,
+        )
+        await _restore_rows(
+            conn,
+            "knowledge_pages",
+            _order_knowledge_pages(parsed.bank_rows.get("knowledge_pages", [])),
             bank_rows_json_encoding=bank_rows_json_encoding,
         )
         # Restored after mental_models so the (mental_model_id, bank_id) FK resolves.