diff --git a/dist/src/store.js b/dist/src/store.js index f6936cb5..f345922d 100644 --- a/dist/src/store.js +++ b/dist/src/store.js @@ -1634,7 +1634,7 @@ export class MemoryStore { const entry = { id: row.id, text: row.text, - vector: row.vector, + vector: toNumberVector(row.vector), category: row.category, scope: rowScope, importance: clampImportance(Number(row.importance)), @@ -1693,7 +1693,7 @@ export class MemoryStore { const entry = { id: row.id, text: row.text, - vector: row.vector, + vector: toNumberVector(row.vector), category: row.category, scope: rowScope, importance: clampImportance(Number(row.importance)), @@ -2727,7 +2727,7 @@ export class MemoryStore { .map((row) => ({ id: row.id, text: row.text, - vector: Array.isArray(row.vector) ? row.vector : [], + vector: toNumberVector(row.vector), category: row.category, scope: row.scope ?? "global", importance: clampImportance(Number(row.importance)), diff --git a/package.json b/package.json index 6abe22cd..e07e666d 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs && node --test test/typed-array-vector-fetch.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", "test:llm-clients-and-auth": "node scripts/run-ci-tests.mjs --group llm-clients-and-auth", "test:packaging-and-workflow": "node scripts/verify-ci-test-manifest.mjs && node scripts/run-ci-tests.mjs --group packaging-and-workflow", "build": "tsc -p tsconfig.json", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 71c59d75..a979f599 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -117,6 +117,7 @@ export const CI_TEST_MANIFEST = [ { group: "storage-and-schema", runner: "node", file: "test/smart-metadata-source-classification.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/scope-owner-leak-hardening.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/isOwnedByAgent.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/typed-array-vector-fetch.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { diff --git a/src/store.ts b/src/store.ts index 3acd3f55..d095f100 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1979,7 +1979,7 @@ export class MemoryStore { const entry: MemoryEntry = { id: row.id as string, text: row.text as string, - vector: row.vector as number[], + vector: toNumberVector(row.vector), category: row.category as MemoryEntry["category"], scope: rowScope, importance: clampImportance(Number(row.importance)), @@ -2045,7 +2045,7 @@ export class MemoryStore { const entry: MemoryEntry = { id: row.id as string, text: row.text as string, - vector: row.vector as number[], + vector: toNumberVector(row.vector), category: row.category as MemoryEntry["category"], scope: rowScope, importance: clampImportance(Number(row.importance)), @@ -3246,7 +3246,7 @@ export class MemoryStore { (row): MemoryEntry => ({ id: row.id as string, text: row.text as string, - vector: Array.isArray(row.vector) ? (row.vector as number[]) : [], + vector: toNumberVector(row.vector), category: row.category as MemoryEntry["category"], scope: (row.scope as string | undefined) ?? "global", importance: clampImportance(Number(row.importance)), diff --git a/test/typed-array-vector-fetch.test.mjs b/test/typed-array-vector-fetch.test.mjs new file mode 100644 index 00000000..aefa52d6 --- /dev/null +++ b/test/typed-array-vector-fetch.test.mjs @@ -0,0 +1,79 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +async function withTempStore(fn) { + const { MemoryStore } = jiti("../src/store.ts"); + const dir = mkdtempSync(join(tmpdir(), "typed-array-vector-")); + const store = new MemoryStore({ dbPath: dir, vectorDim: 8 }); + try { + await fn(store); + } finally { + await store.destroy(); + rmSync(dir, { recursive: true, force: true }); + } +} + +describe("typed-array vectors surviving a real LanceDB round-trip", () => { + it("fetchForCompaction returns the stored vector, not an empty array", async () => { + await withTempStore(async (store) => { + await store.store({ + text: "vector round-trip check", + vector: new Array(8).fill(0.5), + category: "fact", + scope: "global", + importance: 0.5, + metadata: "{}", + }); + + const rows = await store.fetchForCompaction(Date.now() + 10_000, undefined, 100); + assert.equal(rows.length, 1); + assert.equal(rows[0].vector.length, 8, "vector must round-trip with its real dimensionality, not collapse to []"); + assert.ok(Array.isArray(rows[0].vector), "vector must be a real Array, not a typed array"); + assert.deepEqual(rows[0].vector, new Array(8).fill(0.5)); + }); + }); + + it("bm25Search returns a real Array for entry.vector, not a typed array", async () => { + await withTempStore(async (store) => { + await store.store({ + text: "searchable unique keyword xyzzy", + vector: new Array(8).fill(0.25), + category: "fact", + scope: "global", + importance: 0.5, + metadata: "{}", + }); + + const results = await store.bm25Search("xyzzy", 5); + assert.equal(results.length, 1); + assert.ok(Array.isArray(results[0].entry.vector), "vector must be a real Array, not a typed array"); + assert.equal(results[0].entry.vector.length, 8); + assert.deepEqual(results[0].entry.vector, new Array(8).fill(0.25)); + }); + }); + + it("lexicalFallbackSearch returns a real Array for entry.vector, not a typed array", async () => { + await withTempStore(async (store) => { + await store.store({ + text: "searchable unique keyword zyzzx", + vector: new Array(8).fill(0.75), + category: "fact", + scope: "global", + importance: 0.5, + metadata: "{}", + }); + + const results = await store.lexicalFallbackSearch("zyzzx", 5); + assert.equal(results.length, 1); + assert.ok(Array.isArray(results[0].entry.vector), "vector must be a real Array, not a typed array"); + assert.equal(results[0].entry.vector.length, 8); + assert.deepEqual(results[0].entry.vector, new Array(8).fill(0.75)); + }); + }); +});