From da6a0ede589ad1ac9b697fcd8fa11a24a25c2fcc Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Mon, 22 Jun 2026 10:28:35 +0000 Subject: [PATCH 01/39] feat: add hivemind_docs schema and config --- src/config.ts | 5 +++ src/deeplake-schema.ts | 34 +++++++++++++++++++ tests/claude-code/flush-memory-wiring.test.ts | 2 +- tests/claude-code/flush-memory.test.ts | 2 +- tests/claude-code/skillify-auto-pull.test.ts | 1 + tests/claude-code/spawn-wiki-worker.test.ts | 1 + tests/shared/graph/deeplake-pull.test.ts | 1 + tests/shared/graph/deeplake-push.test.ts | 1 + 8 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index cea70804..2d1a10aa 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,6 +15,7 @@ export interface Config { rulesTableName: string; goalsTableName: string; kpisTableName: string; + docsTableName: string; codebaseTableName: string; memoryPath: string; } @@ -70,6 +71,10 @@ export function loadConfig(): Config | null { // table shape. goalsTableName: process.env.HIVEMIND_GOALS_TABLE ?? "hivemind_goals", kpisTableName: process.env.HIVEMIND_KPIS_TABLE ?? "hivemind_kpis", + // Per-file documentation kept fresh on code deltas. INSERT-only + // version-bumped table (see DOCS_COLUMNS in deeplake-schema.ts); the + // VFS path classifier maps memory/docs//.md → row. + docsTableName: process.env.HIVEMIND_DOCS_TABLE ?? "hivemind_docs", codebaseTableName: process.env.HIVEMIND_CODEBASE_TABLE ?? "codebase", memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join(home, ".deeplake", "memory"), }; diff --git a/src/deeplake-schema.ts b/src/deeplake-schema.ts index a47ec2f4..8d20a9f4 100644 --- a/src/deeplake-schema.ts +++ b/src/deeplake-schema.ts @@ -174,6 +174,39 @@ export const KPIS_COLUMNS: readonly ColumnDef[] = Object.freeze([ { name: "plugin_version", sql: "TEXT NOT NULL DEFAULT ''" }, ]); +/** + * Docs table — per-file internal documentation kept fresh on code deltas. + * + * One row per doc version. Edits INSERT a fresh row with version+1; reads + * pick the latest per doc_id (ORDER BY version DESC LIMIT 1). Same + * immutable / version-bumped pattern as RULES_COLUMNS / SKILLS_COLUMNS — + * sidesteps the Deeplake UPDATE-coalescing quirk that bit the wiki worker. + * + * `doc_id` is the stable key = the documented source file path (e.g. + * `src/shell/deeplake-fs.ts`); `path` is the VFS location the doc is read + * from (`/docs//.md`). `anchors` is a JSON array of + * `{ symbol_id, content_hash }` pairs tying doc sections to graph nodes — + * a changed `content_hash` is the objective drift signal that marks the + * doc stale. `tier` is `fast` (per-file, regenerated freely on delta) or + * `slow` (project knowledge, append-only through the gate; never silently + * overwritten by a fast edit). + */ +export const DOCS_COLUMNS: readonly ColumnDef[] = Object.freeze([ + { name: "id", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "doc_id", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "path", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "content", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "anchors", sql: "TEXT NOT NULL DEFAULT '[]'" }, + { name: "tier", sql: "TEXT NOT NULL DEFAULT 'fast'" }, + { name: "status", sql: "TEXT NOT NULL DEFAULT 'active'" }, + { name: "project", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "version", sql: "BIGINT NOT NULL DEFAULT 1" }, + { name: "created_at", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "updated_at", sql: "TEXT NOT NULL DEFAULT ''" }, + { name: "agent", sql: "TEXT NOT NULL DEFAULT 'manual'" }, + { name: "plugin_version", sql: "TEXT NOT NULL DEFAULT ''" }, +]); + // ── Module-load lint ──────────────────────────────────────────────────────── /** @@ -249,6 +282,7 @@ validateSchema("SKILLS_COLUMNS", SKILLS_COLUMNS); validateSchema("RULES_COLUMNS", RULES_COLUMNS); validateSchema("GOALS_COLUMNS", GOALS_COLUMNS); validateSchema("KPIS_COLUMNS", KPIS_COLUMNS); +validateSchema("DOCS_COLUMNS", DOCS_COLUMNS); validateSchema("CODEBASE_COLUMNS", CODEBASE_COLUMNS); // ── SQL builders ──────────────────────────────────────────────────────────── diff --git a/tests/claude-code/flush-memory-wiring.test.ts b/tests/claude-code/flush-memory-wiring.test.ts index 9a9dcdb9..ab93df4e 100644 --- a/tests/claude-code/flush-memory-wiring.test.ts +++ b/tests/claude-code/flush-memory-wiring.test.ts @@ -39,7 +39,7 @@ const NOW = "2026-06-16T00:00:00.000Z"; const cfg: Config = { token: "t", orgId: "o", orgName: "Org", userName: "u", workspaceId: "w", apiUrl: "http://x", tableName: "mem", sessionsTableName: "s", skillsTableName: "sk", rulesTableName: "r", - goalsTableName: "g", kpisTableName: "k", codebaseTableName: "c", memoryPath: "/m", + goalsTableName: "g", kpisTableName: "k", docsTableName: "d", codebaseTableName: "c", memoryPath: "/m", }; beforeEach(() => { diff --git a/tests/claude-code/flush-memory.test.ts b/tests/claude-code/flush-memory.test.ts index 48db9ff9..458cee01 100644 --- a/tests/claude-code/flush-memory.test.ts +++ b/tests/claude-code/flush-memory.test.ts @@ -25,7 +25,7 @@ const NOW = "2026-06-16T00:00:00.000Z"; const fakeConfig: Config = { token: "t", orgId: "o", orgName: "OrgName", userName: "user", workspaceId: "w", apiUrl: "http://x", tableName: "memtable", sessionsTableName: "s", skillsTableName: "sk", - rulesTableName: "r", goalsTableName: "g", kpisTableName: "k", codebaseTableName: "c", + rulesTableName: "r", goalsTableName: "g", kpisTableName: "k", docsTableName: "d", codebaseTableName: "c", memoryPath: "/m", }; diff --git a/tests/claude-code/skillify-auto-pull.test.ts b/tests/claude-code/skillify-auto-pull.test.ts index f0294004..72405589 100644 --- a/tests/claude-code/skillify-auto-pull.test.ts +++ b/tests/claude-code/skillify-auto-pull.test.ts @@ -73,6 +73,7 @@ function makeConfig(): Config { rulesTableName: "hivemind_rules", goalsTableName: "hivemind_goals", kpisTableName: "hivemind_kpis", + docsTableName: "hivemind_docs", codebaseTableName: "codebase", memoryPath: join(tmpHome, ".deeplake", "memory"), }; diff --git a/tests/claude-code/spawn-wiki-worker.test.ts b/tests/claude-code/spawn-wiki-worker.test.ts index 0fd315d3..63e2282b 100644 --- a/tests/claude-code/spawn-wiki-worker.test.ts +++ b/tests/claude-code/spawn-wiki-worker.test.ts @@ -129,6 +129,7 @@ function fakeConfig(): Config { rulesTableName: "hivemind_rules", goalsTableName: "hivemind_goals", kpisTableName: "hivemind_kpis", + docsTableName: "hivemind_docs", codebaseTableName: "codebase", memoryPath: "/tmp/fake-memory", }; diff --git a/tests/shared/graph/deeplake-pull.test.ts b/tests/shared/graph/deeplake-pull.test.ts index 189051f3..4c8f48c7 100644 --- a/tests/shared/graph/deeplake-pull.test.ts +++ b/tests/shared/graph/deeplake-pull.test.ts @@ -31,6 +31,7 @@ function makeConfig(): Config { rulesTableName: "hivemind_rules", goalsTableName: "hivemind_goals", kpisTableName: "hivemind_kpis", + docsTableName: "hivemind_docs", codebaseTableName: "codebase_test", memoryPath: "/tmp/mem", }; diff --git a/tests/shared/graph/deeplake-push.test.ts b/tests/shared/graph/deeplake-push.test.ts index 52c4a756..11ba4aab 100644 --- a/tests/shared/graph/deeplake-push.test.ts +++ b/tests/shared/graph/deeplake-push.test.ts @@ -20,6 +20,7 @@ function makeConfig(): Config { rulesTableName: "hivemind_rules", goalsTableName: "hivemind_goals", kpisTableName: "hivemind_kpis", + docsTableName: "hivemind_docs", codebaseTableName: "codebase_test", memoryPath: "/tmp/mem", }; From 30a87e02ddf121c679097a3580a403ba5f52b964 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Mon, 22 Jun 2026 10:32:05 +0000 Subject: [PATCH 02/39] feat: add insert-only hivemind_docs store --- src/docs/index.ts | 19 ++ src/docs/read.ts | 179 +++++++++++++++++++ src/docs/write.ts | 189 ++++++++++++++++++++ tests/shared/docs.test.ts | 363 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 750 insertions(+) create mode 100644 src/docs/index.ts create mode 100644 src/docs/read.ts create mode 100644 src/docs/write.ts create mode 100644 tests/shared/docs.test.ts diff --git a/src/docs/index.ts b/src/docs/index.ts new file mode 100644 index 00000000..2ee83139 --- /dev/null +++ b/src/docs/index.ts @@ -0,0 +1,19 @@ +/** + * Barrel for `src/docs/`. + * + * Consumers (CLI handler, doc-worker, future VFS read routing) import only + * from this entry point so internal restructuring stays a non-breaking + * change for callers. + */ + +export { insertDoc, editDoc, _MAX_CONTENT_LENGTH } from "./write.js"; +export type { InsertDocInput, EditDocInput, WriteResult } from "./write.js"; + +export { listDocs, getDocLatest, parseAnchors } from "./read.js"; +export type { + DocRow, + DocAnchor, + DocTier, + ListDocsOpts, + QueryFn, +} from "./read.js"; diff --git a/src/docs/read.ts b/src/docs/read.ts new file mode 100644 index 00000000..7313a32e --- /dev/null +++ b/src/docs/read.ts @@ -0,0 +1,179 @@ +/** + * Read helpers for `hivemind_docs`. + * + * The table is append-only with a per-doc version monotone (see ./write.ts). + * Reads always pick the latest row per `doc_id`. Same shape as `src/rules/` — + * v1 fetches the candidate rows and deduplicates in JS, portable across + * whatever subset of Postgres window functions Deeplake exposes and fast + * enough at the expected per-repo scale (file counts in the hundreds). + * + * `doc_id` is the stable key = the documented source file path (e.g. + * `src/shell/deeplake-fs.ts`). `anchors` is persisted as a JSON string and + * re-parsed here into `DocAnchor[]`; a malformed value degrades to `[]` + * rather than throwing, so a single bad row never poisons a list read. + */ + +import { sqlIdent, sqlStr } from "../utils/sql.js"; + +export type QueryFn = (sql: string) => Promise>>; + +/** Doc tier — `fast` per-file docs vs `slow` protected project knowledge. */ +export type DocTier = "fast" | "slow"; + +/** One anchor tying a doc to a graph node + a hash of the code it describes. */ +export interface DocAnchor { + /** Graph node id: `::`. */ + symbol_id: string; + /** sha256 of the symbol's source slice when the doc was last written. */ + content_hash: string; +} + +/** Shape of one row in `hivemind_docs` — mirrors DOCS_COLUMNS exactly. */ +export interface DocRow { + id: string; + doc_id: string; + path: string; + content: string; + anchors: DocAnchor[]; + tier: DocTier; + status: string; + project: string; + version: number; + created_at: string; + updated_at: string; + agent: string; + plugin_version: string; +} + +export interface ListDocsOpts { + /** Filter by status. Default 'active'. Pass 'all' for everything. */ + status?: "active" | "archived" | "all"; + /** Filter to one project. Omit for all projects. */ + project?: string; + /** Max rows returned. Default 200. */ + limit?: number; +} + +const SELECT_COLS = + "id, doc_id, path, content, anchors, tier, status, project, version, " + + "created_at, updated_at, agent, plugin_version"; + +/** + * Return the latest version row for every distinct `doc_id`, filtered by + * status (and optionally project), capped at `limit`. The "latest per id" + * dedup happens in JS — see module docstring for the rationale. + * + * Newest-first ordering (by `updated_at` of the winning version) so a + * caller listing docs sees the most-recently-refreshed files first. + */ +export async function listDocs( + query: QueryFn, + tableName: string, + opts: ListDocsOpts = {}, +): Promise { + const safe = sqlIdent(tableName); + // ORDER BY version DESC primes the JS dedup: the first row seen per + // doc_id is the winning latest-version row. updated_at DESC then id DESC + // are deterministic tie-breakers so listDocs and getDocLatest agree on + // the winner row-for-row under same-millisecond / race duplicates. + const rows = await query( + `SELECT ${SELECT_COLS} FROM "${safe}" ORDER BY version DESC, updated_at DESC, id DESC`, + ); + + const latest = new Map(); + for (const r of rows) { + const row = normalize(r); + if (!row) continue; + if (!latest.has(row.doc_id)) latest.set(row.doc_id, row); + } + + const statusFilter = opts.status ?? "active"; + const filtered = [...latest.values()].filter(r => { + if (statusFilter !== "all" && r.status !== statusFilter) return false; + if (opts.project !== undefined && r.project !== opts.project) return false; + return true; + }); + + filtered.sort( + (a, b) => b.updated_at.localeCompare(a.updated_at) || b.id.localeCompare(a.id), + ); + return filtered.slice(0, opts.limit ?? 200); +} + +/** + * Return the latest version of a single doc by `doc_id`, or `null` if it + * does not exist. Used by `editDoc` in ./write.ts to carry over the prior + * content / immutable `created_at` when the caller omits a field. + */ +export async function getDocLatest( + query: QueryFn, + tableName: string, + docId: string, +): Promise { + const safe = sqlIdent(tableName); + // version DESC picks the highest version; updated_at DESC then id DESC + // break ties deterministically when concurrent edits produced duplicate + // v=N+1 rows for the same doc_id — keeps this in agreement with listDocs. + const rows = await query( + `SELECT ${SELECT_COLS} FROM "${safe}" ` + + `WHERE doc_id = '${sqlStr(docId)}' ` + + `ORDER BY version DESC, updated_at DESC, id DESC LIMIT 1`, + ); + if (rows.length === 0) return null; + return normalize(rows[0]); +} + +/** Parse the `anchors` JSON cell into a typed array; degrade to [] on garbage. */ +export function parseAnchors(raw: unknown): DocAnchor[] { + let arr: unknown = raw; + if (typeof raw === "string") { + if (raw.trim() === "") return []; + try { + arr = JSON.parse(raw); + } catch { + return []; + } + } + if (!Array.isArray(arr)) return []; + const out: DocAnchor[] = []; + for (const item of arr) { + if (item && typeof item === "object") { + const sid = (item as Record).symbol_id; + const hash = (item as Record).content_hash; + if (typeof sid === "string" && typeof hash === "string") { + out.push({ symbol_id: sid, content_hash: hash }); + } + } + } + return out; +} + +/** + * Coerce a row from the Deeplake API client into a typed DocRow. The client + * returns `Record` because it has no schema awareness — + * this is where we re-attach the static type and parse `anchors`. + */ +function normalize(row: Record): DocRow | null { + // version arrives as number (parsed by the client) or string (raw cell). + // Normalize to number; a NaN means the row was malformed and we drop it. + const vRaw = row.version; + const version = + typeof vRaw === "number" ? vRaw : typeof vRaw === "string" ? Number(vRaw) : NaN; + if (!Number.isFinite(version)) return null; + const tier = String(row.tier ?? "fast"); + return { + id: String(row.id ?? ""), + doc_id: String(row.doc_id ?? ""), + path: String(row.path ?? ""), + content: String(row.content ?? ""), + anchors: parseAnchors(row.anchors), + tier: tier === "slow" ? "slow" : "fast", + status: String(row.status ?? ""), + project: String(row.project ?? ""), + version, + created_at: String(row.created_at ?? ""), + updated_at: String(row.updated_at ?? ""), + agent: String(row.agent ?? ""), + plugin_version: String(row.plugin_version ?? ""), + }; +} diff --git a/src/docs/write.ts b/src/docs/write.ts new file mode 100644 index 00000000..ffd98c6b --- /dev/null +++ b/src/docs/write.ts @@ -0,0 +1,189 @@ +/** + * Write helpers for `hivemind_docs` — INSERT-only against the immutable + * skills/rules-table pattern. Every edit appends a fresh row with version+1; + * we never UPDATE. Reads (see ./read.ts) pick the latest version per doc_id. + * + * Why no UPDATEs: the Deeplake backend silently coalesces two rapid UPDATEs + * on the same row (see CLAUDE.md "UPDATE coalescing quirk"). INSERT-only + * sidesteps the bug entirely. See `src/rules/write.ts` for the precedent and + * `deeplake-schema.ts` DOCS_COLUMNS for the column list. + * + * Differences from rules: + * - `doc_id` is the documented source file path, supplied by the caller + * (the file path IS the stable identity), not a generated UUID. + * - `content` is markdown and MAY contain newlines — they are preserved, + * not rejected (unlike rule bodies, which are single-line). + * - `created_at` is an immutable creation timestamp carried across every + * version bump; only `updated_at` advances. This mirrors the + * timestamp-preservation pattern used by goals/skills. + */ + +import { randomUUID } from "node:crypto"; +import { sqlIdent, sqlStr } from "../utils/sql.js"; +import type { DocAnchor, DocRow, DocTier, QueryFn } from "./read.js"; +import { getDocLatest } from "./read.js"; + +export interface InsertDocInput { + /** Documented source file path, e.g. `src/shell/deeplake-fs.ts`. Stable key. */ + doc_id: string; + /** VFS path the doc is read from, e.g. `/docs//.md`. */ + path: string; + /** Markdown body. */ + content: string; + /** Anchors tying doc sections to graph nodes. Default []. */ + anchors?: DocAnchor[]; + /** `fast` (per-file, default) or `slow` (protected project knowledge). */ + tier?: DocTier; + /** Project key the doc belongs to. Empty string lands the column default. */ + project?: string; + /** Override the `agent` column. Default "manual". */ + agent?: string; + /** Plugin version that produced the write. Empty string lands the default. */ + plugin_version?: string; +} + +export interface EditDocInput { + /** Stable doc_id (the source file path). */ + doc_id: string; + /** New markdown body. Omit to keep the previous content. */ + content?: string; + /** New anchors. Omit to keep the previous anchors. */ + anchors?: DocAnchor[]; + /** New tier. Omit to keep the previous tier. */ + tier?: DocTier; + /** New status. Omit to keep the previous status. */ + status?: "active" | "archived"; + /** New VFS path. Omit to keep the previous path. */ + path?: string; + agent?: string; + plugin_version?: string; +} + +export interface WriteResult { + doc_id: string; + version: number; +} + +const MAX_CONTENT_LENGTH = 50_000; + +/** + * Validate the markdown body. Throws on empty input or over-cap length. + * Newlines are allowed — docs are multi-line by nature. + */ +function assertValidContent(content: string): void { + if (content.length === 0) throw new Error("Doc content must not be empty"); + if (content.length > MAX_CONTENT_LENGTH) { + throw new Error( + `Doc content exceeds ${MAX_CONTENT_LENGTH} chars (got ${content.length})`, + ); + } +} + +/** Serialize anchors to a JSON string for the `anchors` TEXT column. */ +function serializeAnchors(anchors: DocAnchor[]): string { + return JSON.stringify( + anchors.map(a => ({ symbol_id: a.symbol_id, content_hash: a.content_hash })), + ); +} + +/** + * Insert a brand new per-file doc at version=1. `created_at` and + * `updated_at` are both stamped now; later edits carry `created_at` forward. + */ +export async function insertDoc( + query: QueryFn, + tableName: string, + input: InsertDocInput, +): Promise { + assertValidContent(input.content); + if (input.doc_id.length === 0) throw new Error("Doc doc_id must not be empty"); + const safe = sqlIdent(tableName); + const rowId = randomUUID(); + const now = new Date().toISOString(); + const anchors = serializeAnchors(input.anchors ?? []); + const tier: DocTier = input.tier ?? "fast"; + + const sql = + `INSERT INTO "${safe}" ` + + `(id, doc_id, path, content, anchors, tier, status, project, version, ` + + `created_at, updated_at, agent, plugin_version) ` + + `VALUES (` + + `'${sqlStr(rowId)}', ` + + `'${sqlStr(input.doc_id)}', ` + + `'${sqlStr(input.path)}', ` + + `E'${sqlStr(input.content)}', ` + + `E'${sqlStr(anchors)}', ` + + `'${sqlStr(tier)}', ` + + `'active', ` + + `'${sqlStr(input.project ?? "")}', ` + + `1, ` + + `'${sqlStr(now)}', ` + + `'${sqlStr(now)}', ` + + `'${sqlStr(input.agent ?? "manual")}', ` + + `'${sqlStr(input.plugin_version ?? "")}'` + + `)`; + await query(sql); + return { doc_id: input.doc_id, version: 1 }; +} + +/** + * Edit an existing doc. Reads the latest version, then INSERTs a new row + * with version+1 carrying the merged fields (omitted fields inherit from + * the prior version). The immutable `created_at` is carried forward; + * `updated_at` advances to now. Throws when the `doc_id` does not exist. + */ +export async function editDoc( + query: QueryFn, + tableName: string, + input: EditDocInput, +): Promise { + const previous = await getDocLatest(query, tableName, input.doc_id); + if (!previous) { + throw new Error(`Doc not found: ${input.doc_id}`); + } + return appendVersion(query, tableName, previous, input); +} + +async function appendVersion( + query: QueryFn, + tableName: string, + previous: DocRow, + next: EditDocInput, +): Promise { + const content = next.content ?? previous.content; + assertValidContent(content); + const safe = sqlIdent(tableName); + const rowId = randomUUID(); + const now = new Date().toISOString(); + const nextVersion = previous.version + 1; + const anchors = serializeAnchors(next.anchors ?? previous.anchors); + const tier: DocTier = next.tier ?? previous.tier; + const status = next.status ?? (previous.status as "active" | "archived"); + const path = next.path ?? previous.path; + + const sql = + `INSERT INTO "${safe}" ` + + `(id, doc_id, path, content, anchors, tier, status, project, version, ` + + `created_at, updated_at, agent, plugin_version) ` + + `VALUES (` + + `'${sqlStr(rowId)}', ` + + `'${sqlStr(previous.doc_id)}', ` + + `'${sqlStr(path)}', ` + + `E'${sqlStr(content)}', ` + + `E'${sqlStr(anchors)}', ` + + `'${sqlStr(tier)}', ` + + `'${sqlStr(status)}', ` + + `'${sqlStr(previous.project)}', ` + + `${nextVersion}, ` + + // created_at carried from the original row — immutable creation stamp. + `'${sqlStr(previous.created_at)}', ` + + `'${sqlStr(now)}', ` + + `'${sqlStr(next.agent ?? "manual")}', ` + + `'${sqlStr(next.plugin_version ?? "")}'` + + `)`; + await query(sql); + return { doc_id: previous.doc_id, version: nextVersion }; +} + +/** Test-only export so unit tests can verify the cap without monkey-patching. */ +export const _MAX_CONTENT_LENGTH = MAX_CONTENT_LENGTH; diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts new file mode 100644 index 00000000..a76161f0 --- /dev/null +++ b/tests/shared/docs.test.ts @@ -0,0 +1,363 @@ +import { describe, expect, it, beforeEach, vi } from "vitest"; +import { + insertDoc, + editDoc, + listDocs, + getDocLatest, + parseAnchors, + _MAX_CONTENT_LENGTH, + type DocRow, +} from "../../src/docs/index.js"; + +/** + * Mock query helper. Each script step receives the SQL and returns rows + * (or throws). The harness captures every SQL string for shape + count + * assertions — see CLAUDE.md "mock the network boundary, not the module + * under test" for the rationale. + */ +function mockQuery(script: Array<(sql: string) => unknown>) { + const calls: string[] = []; + let step = 0; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + if (step < script.length) { + const out = script[step++](sql); + return Array.isArray(out) ? (out as Array>) : []; + } + return []; + }); + return { calls, query }; +} + +const TBL = "hivemind_docs"; + +/** + * Build a fake row matching DOCS_COLUMNS shape as the Deeplake client + * returns it — note `anchors` arrives as a JSON STRING (TEXT column). + */ +function fakeRow(overrides: Partial> = {}): Record { + return { + id: "row-uuid", + doc_id: "src/shell/deeplake-fs.ts", + path: "/docs/myproj/deeplake-fs.ts.md", + content: "# deeplake-fs\n\nThe VFS.", + anchors: JSON.stringify([{ symbol_id: "src/shell/deeplake-fs.ts:readFile:function", content_hash: "abc123" }]), + tier: "fast", + status: "active", + project: "myproj", + version: 1, + created_at: "2026-05-20T10:00:00.000Z", + updated_at: "2026-05-20T10:00:00.000Z", + agent: "manual", + plugin_version: "0.7.105", + ...overrides, + }; +} + +beforeEach(() => { + vi.useRealTimers(); +}); + +// ── insertDoc ───────────────────────────────────────────────────────────────── + +describe("insertDoc", () => { + it("INSERTs a v1 row, both timestamps equal, content + anchors as E-strings", async () => { + const { calls, query } = mockQuery([() => []]); + const result = await insertDoc(query, TBL, { + doc_id: "src/shell/deeplake-fs.ts", + path: "/docs/myproj/deeplake-fs.ts.md", + content: "# deeplake-fs", + anchors: [{ symbol_id: "src/shell/deeplake-fs.ts:readFile:function", content_hash: "abc123" }], + project: "myproj", + }); + expect(result).toEqual({ doc_id: "src/shell/deeplake-fs.ts", version: 1 }); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatch(/^INSERT INTO "hivemind_docs"/); + // version literal is 1, never quoted + expect(calls[0]).toMatch(/, 1, /); + expect(calls[0]).toContain("'active'"); + expect(calls[0]).toContain("'myproj'"); + // content + anchors use E-string literals so backslashes / quotes stay safe + expect(calls[0]).toContain(`E'# deeplake-fs'`); + expect(calls[0]).toContain(`E'[{"symbol_id":"src/shell/deeplake-fs.ts:readFile:function","content_hash":"abc123"}]'`); + // tier defaults to fast on first insert + expect(calls[0]).toContain("'fast'"); + // created_at and updated_at are stamped identically at insert time + const stamps = calls[0].match(/'(\d{4}-\d{2}-\d{2}T[\d:.]+Z)'/g); + expect(stamps).not.toBeNull(); + expect(stamps!.length).toBeGreaterThanOrEqual(2); + expect(stamps![0]).toBe(stamps![1]); + }); + + it("PRESERVES newlines in markdown content (docs are multi-line, unlike rules)", async () => { + const { calls, query } = mockQuery([() => []]); + await insertDoc(query, TBL, { + doc_id: "a.ts", + path: "/docs/p/a.ts.md", + content: "# Title\n\n- bullet one\n- bullet two", + }); + // Newlines survive into the E-string body — no rejection, no mangling. + expect(calls[0]).toContain("# Title\n\n- bullet one\n- bullet two"); + }); + + it("escapes SQL-special characters in content and anchors", async () => { + const { calls, query } = mockQuery([() => []]); + await insertDoc(query, TBL, { + doc_id: "a.ts", + path: "/docs/p/a.ts.md", + content: "don't `rm -rf /` \\ ever", + }); + expect(calls[0]).toContain(`E'don''t \`rm -rf /\` \\\\ ever'`); + }); + + it("defaults anchors to [] and tier to fast when omitted", async () => { + const { calls, query } = mockQuery([() => []]); + await insertDoc(query, TBL, { doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "x" }); + expect(calls[0]).toContain(`E'[]'`); + expect(calls[0]).toContain("'fast'"); + expect(calls[0]).toContain("'manual'"); + }); + + it("writes the slow tier when requested", async () => { + const { calls, query } = mockQuery([() => []]); + await insertDoc(query, TBL, { doc_id: "_project", path: "/docs/p/_project.md", content: "x", tier: "slow" }); + expect(calls[0]).toContain("'slow'"); + }); + + it("rejects empty content", async () => { + const { calls, query } = mockQuery([() => []]); + await expect( + insertDoc(query, TBL, { doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "" }), + ).rejects.toThrow(/must not be empty/); + expect(calls).toHaveLength(0); + }); + + it("rejects empty doc_id", async () => { + const { calls, query } = mockQuery([() => []]); + await expect( + insertDoc(query, TBL, { doc_id: "", path: "/docs/p/a.ts.md", content: "x" }), + ).rejects.toThrow(/doc_id must not be empty/); + expect(calls).toHaveLength(0); + }); + + it(`rejects content longer than ${_MAX_CONTENT_LENGTH} chars`, async () => { + const { calls, query } = mockQuery([() => []]); + const oversized = "x".repeat(_MAX_CONTENT_LENGTH + 1); + await expect( + insertDoc(query, TBL, { doc_id: "a.ts", path: "/docs/p/a.ts.md", content: oversized }), + ).rejects.toThrow(/exceeds 50000 chars/); + expect(calls).toHaveLength(0); + }); + + it("rejects SQL-identifier injection in the table name", async () => { + const { query } = mockQuery([() => []]); + await expect( + insertDoc(query, `x"; DROP TABLE y; --`, { doc_id: "a.ts", path: "/p", content: "x" }), + ).rejects.toThrow(); + }); +}); + +// ── editDoc ─────────────────────────────────────────────────────────────────── + +describe("editDoc", () => { + it("reads latest, then INSERTs version+1 carrying the IMMUTABLE created_at while updated_at advances", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ version: 1, content: "old", created_at: "2026-05-20T10:00:00.000Z" })], + () => [], + ]); + const result = await editDoc(query, TBL, { doc_id: "src/shell/deeplake-fs.ts", content: "new" }); + expect(result).toEqual({ doc_id: "src/shell/deeplake-fs.ts", version: 2 }); + expect(calls).toHaveLength(2); + expect(calls[0]).toMatch(/^SELECT .* FROM "hivemind_docs" WHERE doc_id = 'src\/shell\/deeplake-fs.ts' ORDER BY version DESC, updated_at DESC, id DESC LIMIT 1$/); + expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + expect(calls[1]).toContain(`E'new'`); + expect(calls[1]).toContain(", 2, "); + // The original creation stamp is carried forward verbatim... + expect(calls[1]).toContain("'2026-05-20T10:00:00.000Z'"); + // ...and updated_at is a DIFFERENT, newer timestamp (now). + const stamps = calls[1].match(/'(\d{4}-\d{2}-\d{2}T[\d:.]+Z)'/g)!; + expect(stamps[0]).toBe("'2026-05-20T10:00:00.000Z'"); // created_at + expect(stamps[1]).not.toBe("'2026-05-20T10:00:00.000Z'"); // updated_at = now + }); + + it("carries over previous content + anchors when only status changes", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ version: 3, content: "preserve me" })], + () => [], + ]); + const result = await editDoc(query, TBL, { doc_id: "src/shell/deeplake-fs.ts", status: "archived" }); + expect(result.version).toBe(4); + expect(calls[1]).toContain(`E'preserve me'`); + expect(calls[1]).toContain("'archived'"); + // prior anchors round-trip through serialize unchanged + expect(calls[1]).toContain(`content_hash":"abc123`); + }); + + it("replaces anchors when new ones are supplied", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ version: 1 })], + () => [], + ]); + await editDoc(query, TBL, { + doc_id: "src/shell/deeplake-fs.ts", + content: "refreshed", + anchors: [{ symbol_id: "src/shell/deeplake-fs.ts:writeFile:function", content_hash: "def456" }], + }); + expect(calls[1]).toContain("def456"); + expect(calls[1]).not.toContain("abc123"); + }); + + it("throws when doc_id does not exist (SELECT only, no wasted INSERT)", async () => { + const { calls, query } = mockQuery([() => []]); + await expect( + editDoc(query, TBL, { doc_id: "missing.ts", content: "x" }), + ).rejects.toThrow(/Doc not found: missing.ts/); + expect(calls).toHaveLength(1); + }); + + it("rejects empty content on edit, leaving the SELECT-but-no-INSERT trail", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ version: 1 })], + () => [], + ]); + await expect( + editDoc(query, TBL, { doc_id: "src/shell/deeplake-fs.ts", content: "" }), + ).rejects.toThrow(/must not be empty/); + expect(calls).toHaveLength(1); // SELECT only + }); +}); + +// ── listDocs ────────────────────────────────────────────────────────────────── + +describe("listDocs", () => { + it("returns latest version per doc_id, active only, newest-first by updated_at, default limit 200", async () => { + const { calls, query } = mockQuery([ + () => [ + fakeRow({ id: "a2", doc_id: "A", version: 2, content: "A v2", updated_at: "2026-05-20T10:02:00Z" }), + fakeRow({ id: "a1", doc_id: "A", version: 1, content: "A v1", updated_at: "2026-05-20T10:01:00Z" }), + fakeRow({ id: "b1", doc_id: "B", version: 1, content: "B v1", updated_at: "2026-05-20T10:00:00Z" }), + fakeRow({ id: "c1", doc_id: "C", version: 1, status: "archived", content: "C arch", updated_at: "2026-05-20T09:59:00Z" }), + ], + () => [], + ]); + const rows = await listDocs(query, TBL); + expect(rows.map(r => r.doc_id)).toEqual(["A", "B"]); + expect(rows[0].content).toBe("A v2"); + expect(rows[0].version).toBe(2); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatch(/^SELECT .* FROM "hivemind_docs" ORDER BY version DESC, updated_at DESC, id DESC$/); + }); + + it("honors status='all'", async () => { + const { query } = mockQuery([ + () => [ + fakeRow({ doc_id: "A", version: 1, status: "active" }), + fakeRow({ doc_id: "B", version: 1, status: "archived" }), + ], + ]); + const rows = await listDocs(query, TBL, { status: "all" }); + expect(rows.map(r => r.doc_id).sort()).toEqual(["A", "B"]); + }); + + it("filters by project", async () => { + const { query } = mockQuery([ + () => [ + fakeRow({ doc_id: "A", version: 1, project: "p1" }), + fakeRow({ doc_id: "B", version: 1, project: "p2" }), + ], + ]); + const rows = await listDocs(query, TBL, { project: "p2" }); + expect(rows.map(r => r.doc_id)).toEqual(["B"]); + }); + + it("respects the limit parameter", async () => { + const { query } = mockQuery([ + () => Array.from({ length: 25 }, (_, i) => + fakeRow({ + doc_id: `doc-${i}`, + version: 1, + updated_at: `2026-05-20T10:${String(i).padStart(2, "0")}:00Z`, + }), + ), + ]); + const rows = await listDocs(query, TBL, { limit: 3 }); + expect(rows).toHaveLength(3); + expect(rows[0].doc_id).toBe("doc-24"); // newest by updated_at + }); + + it("drops malformed rows (NaN version) silently", async () => { + const { query } = mockQuery([ + () => [ + fakeRow({ doc_id: "good", version: 1 }), + { doc_id: "bad", version: "not-a-number" }, + ], + ]); + const rows = await listDocs(query, TBL); + expect(rows.map(r => r.doc_id)).toEqual(["good"]); + }); + + it("parses the anchors JSON string back into typed objects", async () => { + const { query } = mockQuery([() => [fakeRow({ doc_id: "A", version: 1 })]]); + const rows = await listDocs(query, TBL); + expect(rows[0].anchors).toEqual([ + { symbol_id: "src/shell/deeplake-fs.ts:readFile:function", content_hash: "abc123" }, + ]); + }); +}); + +// ── getDocLatest ────────────────────────────────────────────────────────────── + +describe("getDocLatest", () => { + it("returns the single latest row, LIMIT 1, escaped doc_id", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ doc_id: "X", version: 5, content: "current" })], + ]); + const row = await getDocLatest(query, TBL, "X"); + expect(row?.version).toBe(5); + expect(row?.content).toBe("current"); + expect(calls[0]).toMatch(/LIMIT 1$/); + expect(calls[0]).toContain(`doc_id = 'X'`); + expect(calls[0]).toContain("ORDER BY version DESC, updated_at DESC, id DESC"); + }); + + it("returns null when nothing matches", async () => { + const { query } = mockQuery([() => []]); + expect(await getDocLatest(query, TBL, "missing")).toBeNull(); + }); + + it("escapes the doc_id in the WHERE clause", async () => { + const { calls, query } = mockQuery([() => []]); + await getDocLatest(query, TBL, "x' OR '1'='1"); + expect(calls[0]).toContain(`doc_id = 'x'' OR ''1''=''1'`); + }); +}); + +// ── parseAnchors ────────────────────────────────────────────────────────────── + +describe("parseAnchors", () => { + it("parses a JSON string into typed anchors", () => { + expect(parseAnchors('[{"symbol_id":"a:b:function","content_hash":"h"}]')).toEqual([ + { symbol_id: "a:b:function", content_hash: "h" }, + ]); + }); + + it("degrades to [] on empty / malformed / non-array input", () => { + expect(parseAnchors("")).toEqual([]); + expect(parseAnchors("not json")).toEqual([]); + expect(parseAnchors('{"not":"array"}')).toEqual([]); + expect(parseAnchors(null)).toEqual([]); + expect(parseAnchors(undefined)).toEqual([]); + }); + + it("filters out items missing symbol_id or content_hash", () => { + expect( + parseAnchors('[{"symbol_id":"a:b:fn","content_hash":"h"},{"symbol_id":"x"},{"content_hash":"y"},42]'), + ).toEqual([{ symbol_id: "a:b:fn", content_hash: "h" }]); + }); + + it("accepts an already-parsed array (defensive)", () => { + expect(parseAnchors([{ symbol_id: "a:b:fn", content_hash: "h" }] as unknown)).toEqual([ + { symbol_id: "a:b:fn", content_hash: "h" }, + ]); + }); +}); From 10443fd557cb0c9fdd9d7083803b2ed5149a0445 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Mon, 22 Jun 2026 13:16:53 +0000 Subject: [PATCH 03/39] fix: add idempotent setDoc to prevent doc history fork --- src/config.ts | 7 +++- src/docs/index.ts | 4 +- src/docs/write.ts | 78 +++++++++++++++++++++++++++++++++++++++ tests/shared/docs.test.ts | 67 +++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 2d1a10aa..1c3cb08d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -72,8 +72,11 @@ export function loadConfig(): Config | null { goalsTableName: process.env.HIVEMIND_GOALS_TABLE ?? "hivemind_goals", kpisTableName: process.env.HIVEMIND_KPIS_TABLE ?? "hivemind_kpis", // Per-file documentation kept fresh on code deltas. INSERT-only - // version-bumped table (see DOCS_COLUMNS in deeplake-schema.ts); the - // VFS path classifier maps memory/docs//.md → row. + // version-bumped table (see DOCS_COLUMNS in deeplake-schema.ts). + // Phase 1: written/read through the `hivemind docs` CLI + worker via the + // src/docs store. NOT yet routed through the VFS path classifier — when + // VFS routing lands it MUST use the INSERT-only store, never the goals + // UPDATE-or-INSERT path (which is vulnerable to UPDATE-coalescing). docsTableName: process.env.HIVEMIND_DOCS_TABLE ?? "hivemind_docs", codebaseTableName: process.env.HIVEMIND_CODEBASE_TABLE ?? "codebase", memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join(home, ".deeplake", "memory"), diff --git a/src/docs/index.ts b/src/docs/index.ts index 2ee83139..81a64429 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -6,8 +6,8 @@ * change for callers. */ -export { insertDoc, editDoc, _MAX_CONTENT_LENGTH } from "./write.js"; -export type { InsertDocInput, EditDocInput, WriteResult } from "./write.js"; +export { insertDoc, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; +export type { InsertDocInput, EditDocInput, SetDocInput, WriteResult } from "./write.js"; export { listDocs, getDocLatest, parseAnchors } from "./read.js"; export type { diff --git a/src/docs/write.ts b/src/docs/write.ts index ffd98c6b..04d6c056 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -42,6 +42,25 @@ export interface InsertDocInput { plugin_version?: string; } +export interface SetDocInput { + /** Documented source file path, e.g. `src/shell/deeplake-fs.ts`. Stable key. */ + doc_id: string; + /** VFS path the doc is read from, e.g. `/docs//.md`. */ + path: string; + /** Markdown body. */ + content: string; + /** Anchors tying doc sections to graph nodes. Default []. */ + anchors?: DocAnchor[]; + /** `fast` (per-file, default) or `slow` (protected project knowledge). */ + tier?: DocTier; + /** Project key the doc belongs to. */ + project?: string; + /** Status to set. Defaults to keeping the previous (or 'active' on first write). */ + status?: "active" | "archived"; + agent?: string; + plugin_version?: string; +} + export interface EditDocInput { /** Stable doc_id (the source file path). */ doc_id: string; @@ -144,6 +163,65 @@ export async function editDoc( return appendVersion(query, tableName, previous, input); } +/** + * Idempotent upsert by `doc_id` — the public entry point CLI / worker code + * should use. Reads the latest version: if the doc exists it appends a new + * version (carrying the immutable `created_at`); if not it inserts v1. + * + * This is what makes `doc_id = file path` safe as a caller-supplied key. + * Calling `insertDoc` directly for an already-documented file would fork + * history into two parallel `version=1` rows — `setDoc` never does, because + * the file path identity always resolves to one version chain. + */ +export async function setDoc( + query: QueryFn, + tableName: string, + input: SetDocInput, +): Promise { + const previous = await getDocLatest(query, tableName, input.doc_id); + if (!previous) { + return insertDoc(query, tableName, { + doc_id: input.doc_id, + path: input.path, + content: input.content, + anchors: input.anchors, + tier: input.tier, + project: input.project, + agent: input.agent, + plugin_version: input.plugin_version, + }); + } + return appendVersion(query, tableName, previous, { + doc_id: input.doc_id, + content: input.content, + anchors: input.anchors, + tier: input.tier, + status: input.status, + path: input.path, + agent: input.agent, + plugin_version: input.plugin_version, + }); +} + +/** + * Archive a doc (soft delete) — appends a version with status='archived', + * preserving content + audit trail. Throws when the doc_id does not exist. + * Used as the delete primitive for the `doc_id = path` lifecycle: when a + * source file is removed, its doc is archived rather than hard-deleted. + */ +export async function archiveDoc( + query: QueryFn, + tableName: string, + input: { doc_id: string; agent?: string; plugin_version?: string }, +): Promise { + return editDoc(query, tableName, { + doc_id: input.doc_id, + status: "archived", + agent: input.agent, + plugin_version: input.plugin_version, + }); +} + async function appendVersion( query: QueryFn, tableName: string, diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index a76161f0..7c7c4695 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it, beforeEach, vi } from "vitest"; import { insertDoc, editDoc, + setDoc, + archiveDoc, listDocs, getDocLatest, parseAnchors, @@ -227,6 +229,71 @@ describe("editDoc", () => { }); }); +// ── setDoc (idempotent upsert — the fork-history fix) ───────────────────────── + +describe("setDoc", () => { + it("INSERTs v1 when the doc_id does not exist yet", async () => { + const { calls, query } = mockQuery([ + () => [], // getDocLatest → none + () => [], // INSERT + ]); + const result = await setDoc(query, TBL, { + doc_id: "src/a.ts", + path: "/docs/p/a.ts.md", + content: "first", + project: "p", + }); + expect(result).toEqual({ doc_id: "src/a.ts", version: 1 }); + expect(calls).toHaveLength(2); + expect(calls[0]).toMatch(/^SELECT .* WHERE doc_id = 'src\/a.ts'/); + expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + expect(calls[1]).toMatch(/, 1, /); + }); + + it("APPENDS version+1 when the doc_id already exists — never forks a second v1", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ doc_id: "src/a.ts", version: 4, created_at: "2026-01-01T00:00:00.000Z" })], + () => [], + ]); + const result = await setDoc(query, TBL, { + doc_id: "src/a.ts", + path: "/docs/p/a.ts.md", + content: "updated", + project: "p", + }); + expect(result).toEqual({ doc_id: "src/a.ts", version: 5 }); + expect(calls).toHaveLength(2); + // The INSERT is version 5, NOT a second version 1 — the fork-history bug + // Codex flagged is impossible through setDoc. + expect(calls[1]).toContain(", 5, "); + expect(calls[1]).not.toMatch(/, 1, /); + // immutable created_at carried from the existing chain + expect(calls[1]).toContain("'2026-01-01T00:00:00.000Z'"); + expect(calls[1]).toContain(`E'updated'`); + }); +}); + +// ── archiveDoc (soft delete primitive) ──────────────────────────────────────── + +describe("archiveDoc", () => { + it("appends a version with status='archived', preserving content", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ doc_id: "src/gone.ts", version: 2, content: "keep me" })], + () => [], + ]); + const result = await archiveDoc(query, TBL, { doc_id: "src/gone.ts" }); + expect(result.version).toBe(3); + expect(calls[1]).toContain("'archived'"); + expect(calls[1]).toContain(`E'keep me'`); + }); + + it("throws when archiving a doc that does not exist", async () => { + const { calls, query } = mockQuery([() => []]); + await expect(archiveDoc(query, TBL, { doc_id: "nope.ts" })).rejects.toThrow(/Doc not found/); + expect(calls).toHaveLength(1); + }); +}); + // ── listDocs ────────────────────────────────────────────────────────────────── describe("listDocs", () => { From 0ce3c1b53cb760cbe6a81c4b584e659000417aed Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Mon, 22 Jun 2026 13:17:01 +0000 Subject: [PATCH 04/39] feat: add ensureDocsTable and hivemind docs command --- src/cli/index.ts | 6 + src/commands/docs.ts | 256 +++++++++++++++++++++++++++++++++++++++++++ src/deeplake-api.ts | 23 ++++ 3 files changed, 285 insertions(+) create mode 100644 src/commands/docs.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index 551299ad..08730c86 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -18,6 +18,7 @@ import { runDashboardCommand } from "../commands/dashboard.js"; import { runSkillifyCommand } from "../commands/skillify.js"; import { runRulesCommand } from "../commands/rules.js"; import { runGoalCommand, runKpiCommand } from "../commands/goal.js"; +import { runDocsCommand } from "../commands/docs.js"; import { runContextCommand } from "../commands/context.js"; import { runBackfillMemory } from "../commands/backfill-memory.js"; import { runFlushMemory } from "../commands/flush-memory.js"; @@ -486,6 +487,11 @@ async function main(): Promise { return; } + if (cmd === "docs" || cmd === "doc") { + await runDocsCommand(args.slice(1)); + return; + } + if (cmd === "context") { await runContextCommand(args.slice(1)); return; diff --git a/src/commands/docs.ts b/src/commands/docs.ts new file mode 100644 index 00000000..6f1fd594 --- /dev/null +++ b/src/commands/docs.ts @@ -0,0 +1,256 @@ +#!/usr/bin/env node + +/** + * CLI surface for `hivemind docs` — per-file documentation kept fresh on + * code deltas (Phase 1: manual set/show/list/archive; the delta worker + * drives `setDoc` programmatically later). + * + * Usage: + * hivemind docs set [""] [--file ] [--project P] [--tier fast|slow] [--path ] + * Idempotent upsert by doc-id (the source file path). First write = + * v1; subsequent writes append v+1, preserving the immutable + * created_at. Content comes from the positional arg, or --file, or + * stdin when the positional is "-". + * hivemind docs show + * Print the latest version's metadata + markdown body. + * hivemind docs list [--project P] [--status active|archived|all] [--limit N] + * List the latest version per doc-id. + * hivemind docs archive + * Soft-delete (status='archived'), preserving content + audit trail. + * + * The handler is deliberately thin — it parses argv, loads config, + * constructs the api client, and delegates to src/docs/{write,read}. All + * SQL escaping and version-bump logic lives in the docs module. + */ + +import { readFileSync } from "node:fs"; +import { loadConfig } from "../config.js"; +import { DeeplakeApi } from "../deeplake-api.js"; +import { getVersion } from "../cli/version.js"; +import { + setDoc, + archiveDoc, + listDocs, + getDocLatest, + type DocRow, + type DocTier, +} from "../docs/index.js"; +import { isMissingTableError } from "../deeplake-schema.js"; + +const USAGE = ` +hivemind docs — per-file documentation kept fresh on code deltas + +Usage: + hivemind docs set [""] [--file ] [--project P] [--tier fast|slow] [--path ] + hivemind docs show + hivemind docs list [--project P] [--status active|archived|all] [--limit N] + hivemind docs archive +`.trim(); + +function requireConfig(): NonNullable> { + const cfg = loadConfig(); + if (!cfg) { + console.error("Not logged in. Run `hivemind login` first."); + process.exit(2); + throw new Error("unreachable"); + } + return cfg; +} + +function makeApi(cfg: NonNullable>): DeeplakeApi { + return new DeeplakeApi(cfg.token, cfg.apiUrl, cfg.orgId, cfg.workspaceId, cfg.tableName); +} + +function flagValue(args: string[], name: string): string | undefined { + const idx = args.findIndex(a => a === name || a.startsWith(`${name}=`)); + if (idx === -1) return undefined; + return args[idx].includes("=") ? args[idx].split("=", 2)[1] : args[idx + 1]; +} + +function parseStatus(args: string[]): "active" | "archived" | "all" { + const raw = flagValue(args, "--status"); + if (raw === undefined) return "active"; + if (raw === "active" || raw === "archived" || raw === "all") return raw; + console.error(`Invalid --status value: ${raw}. Allowed: active | archived | all.`); + process.exit(1); + throw new Error("unreachable"); +} + +function parseTier(args: string[]): DocTier { + const raw = flagValue(args, "--tier"); + if (raw === undefined) return "fast"; + if (raw === "fast" || raw === "slow") return raw; + console.error(`Invalid --tier value: ${raw}. Allowed: fast | slow.`); + process.exit(1); + throw new Error("unreachable"); +} + +function parseLimit(args: string[]): number { + const raw = flagValue(args, "--limit"); + if (raw === undefined) return 200; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0 || !Number.isInteger(n)) { + console.error(`Invalid --limit value: ${raw}. Must be a positive integer.`); + process.exit(1); + throw new Error("unreachable"); + } + return n; +} + +const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit"]); + +/** Drop flag tokens (and their values) so positional scan sees only doc-id / content. */ +function stripKnownFlags(args: string[]): string[] { + const out: string[] = []; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (KNOWN_FLAGS.has(a)) { + i++; + continue; + } + if (KNOWN_FLAGS.has(a.split("=", 2)[0])) continue; + out.push(a); + } + return out; +} + +/** Resolve the markdown body from positional arg, --file, or stdin ("-"). */ +function resolveContent(positionalContent: string | undefined, args: string[]): string { + const file = flagValue(args, "--file"); + if (file !== undefined) return readFileSync(file, "utf-8"); + if (positionalContent === "-") return readFileSync(0, "utf-8"); // stdin + return positionalContent ?? ""; +} + +/** Default VFS path for a doc when --path is omitted. */ +function defaultVfsPath(project: string, docId: string): string { + const proj = project || "default"; + return `/docs/${proj}/${docId}.md`; +} + +function formatListRow(r: DocRow): string { + const tag = r.status === "archived" ? "[archived]" : "[active]"; + const anchors = r.anchors.length === 1 ? "1 anchor" : `${r.anchors.length} anchors`; + return `${tag} ${r.doc_id} v${r.version} (${r.tier}, ${anchors}) ${r.path}`; +} + +export async function runDocsCommand(args: string[]): Promise { + const sub = args[0]; + if (!sub || sub === "--help" || sub === "-h" || sub === "help") { + console.log(USAGE); + return; + } + + const cfg = requireConfig(); + const api = makeApi(cfg); + const tableName = cfg.docsTableName; + const query = api.query.bind(api); + const pluginVersion = getVersion(); + + // Only write subcommands need DDL — read-only show/list fall back to + // isMissingTableError so a fresh-install user doesn't pay a CREATE round-trip. + const WRITE_SUBS = new Set(["set", "archive"]); + if (WRITE_SUBS.has(sub)) { + await api.ensureDocsTable(tableName); + } + + if (sub === "set") { + const positional = stripKnownFlags(args.slice(1)); + const docId = positional[0]; + if (!docId) { + console.error('Missing doc-id. Usage: hivemind docs set "" [--file ]'); + process.exit(1); + throw new Error("unreachable"); + } + const project = flagValue(args, "--project") ?? ""; + const content = resolveContent(positional[1], args); + const path = flagValue(args, "--path") ?? defaultVfsPath(project, docId); + const tier = parseTier(args); + try { + const out = await setDoc(query, tableName, { + doc_id: docId, + path, + content, + tier, + project, + agent: cfg.userName, + plugin_version: pluginVersion, + }); + console.log(`Set doc ${out.doc_id} → v${out.version}.`); + } catch (err) { + console.error(`Set failed: ${(err as Error).message}`); + process.exit(1); + } + return; + } + + if (sub === "show") { + const docId = stripKnownFlags(args.slice(1))[0]; + if (!docId) { + console.error("Usage: hivemind docs show "); + process.exit(1); + throw new Error("unreachable"); + } + let row: DocRow | null = null; + try { + row = await getDocLatest(query, tableName, docId); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + if (!row) { + console.log(`(no doc for ${docId})`); + return; + } + console.log(`# ${row.doc_id}`); + console.log(`version: ${row.version} tier: ${row.tier} status: ${row.status}`); + console.log(`project: ${row.project} path: ${row.path}`); + console.log(`created: ${row.created_at} updated: ${row.updated_at}`); + console.log(`anchors: ${row.anchors.length}`); + console.log("---"); + console.log(row.content); + return; + } + + if (sub === "list") { + const project = flagValue(args, "--project"); + const status = parseStatus(args.slice(1)); + const limit = parseLimit(args.slice(1)); + let rows: DocRow[] = []; + try { + rows = await listDocs(query, tableName, { status, project, limit }); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + if (rows.length === 0) { + console.log(`(no docs with status=${status})`); + return; + } + for (const r of rows) console.log(formatListRow(r)); + return; + } + + if (sub === "archive") { + const docId = stripKnownFlags(args.slice(1))[0]; + if (!docId) { + console.error("Usage: hivemind docs archive "); + process.exit(1); + throw new Error("unreachable"); + } + try { + const out = await archiveDoc(query, tableName, { + doc_id: docId, + agent: cfg.userName, + plugin_version: pluginVersion, + }); + console.log(`Archived doc ${out.doc_id} → v${out.version}.`); + } catch (err) { + console.error(`Archive failed: ${(err as Error).message}`); + process.exit(1); + } + return; + } + + console.error(`Unknown subcommand: ${sub}`); + console.error(USAGE); + process.exit(1); +} diff --git a/src/deeplake-api.ts b/src/deeplake-api.ts index ea1ef8bc..c3296605 100644 --- a/src/deeplake-api.ts +++ b/src/deeplake-api.ts @@ -11,6 +11,7 @@ import { RULES_COLUMNS, GOALS_COLUMNS, KPIS_COLUMNS, + DOCS_COLUMNS, buildCreateTableSql, healMissingColumns, } from "./deeplake-schema.js"; @@ -676,6 +677,28 @@ export class DeeplakeApi { await this.healSchema(safe, KPIS_COLUMNS); await this.ensureLookupIndex(safe, "goal_id_kpi_id", `("goal_id", "kpi_id")`); } + + /** + * Create the docs table — per-file documentation kept fresh on code deltas. + * + * INSERT-only version-bumped (same write pattern as rules/skills): every + * edit appends a fresh row with version+1, reads pick the latest per + * doc_id via `ORDER BY version DESC LIMIT 1` (see src/docs/read.ts). + * Sidesteps the Deeplake UPDATE-coalescing quirk by never UPDATEing. + * The (doc_id, version) index is what the latest-row read scans. + */ + async ensureDocsTable(name: string): Promise { + const safe = sqlIdent(name); + const tables = await this.listTables(); + if (!tables.includes(safe)) { + log(`table "${safe}" not found, creating`); + await this.createTableWithRetry(buildCreateTableSql(safe, DOCS_COLUMNS), safe); + log(`table "${safe}" created`); + if (!tables.includes(safe)) this._tablesCache = [...tables, safe]; + } + await this.healSchema(safe, DOCS_COLUMNS); + await this.ensureLookupIndex(safe, "doc_id_version", `("doc_id", "version")`); + } } // ── Test helpers ──────────────────────────────────────────────────────────── From 2f9518e5344eb7b3b2eab08106975c8cf55bbf07 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:02:22 +0000 Subject: [PATCH 05/39] refactor: extract reusable impactedNodes from renderImpact --- src/graph/render/impact.ts | 82 ++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/src/graph/render/impact.ts b/src/graph/render/impact.ts index 0ded233c..a0b60670 100644 --- a/src/graph/render/impact.ts +++ b/src/graph/render/impact.ts @@ -19,23 +19,23 @@ const IMPACT_CAP = 80; /** Safety bound on BFS depth so a pathological graph can't run away. */ const MAX_DEPTH = 25; -export function renderImpact(snap: GraphSnapshot, pattern: string): string { - const needle = pattern.toLowerCase(); - const matches = snap.nodes.filter((n) => n.id.toLowerCase().includes(needle)); - if (matches.length === 0) { - return `No node matches "${pattern}". Try cat memory/graph/find/${pattern} to explore.`; - } - if (matches.length > 1) { - const lines = [`"${pattern}" matches ${matches.length} nodes — be more specific:`, ""]; - for (const m of matches.slice(0, 20)) lines.push(` ${m.id}`); - if (matches.length > 20) lines.push(` ... and ${matches.length - 20} more`); - return lines.join("\n"); - } - const target = matches[0]!; - - // Reverse adjacency: target -> [edges pointing at it]. Build once. Only edges - // whose SOURCE is a real node are kept, so a dependent is always a graph node - // (defensive consistency with tour/path/neighborhood; codex review). +/** + * Reverse-edge BFS shared by `renderImpact` (single target, for display) and + * `impactedNodes` (many seeds, for doc drift widening). Walks INCOMING + * `calls`/`imports`/`extends`/`implements`/`method_of` edges from the seeds + * to collect every transitive dependent. Deterministic: seeds and each level + * are processed in id order so `viaOf` is stable. + * + * Returns `depthOf` (every reached node incl. seeds at depth 0) and `viaOf` + * (the relation/source that first reached each non-seed node). + */ +function reverseBfs( + snap: GraphSnapshot, + seeds: Iterable, + maxDepth = MAX_DEPTH, +): { depthOf: Map; viaOf: Map } { + // Reverse adjacency: target -> [edges pointing at it]. Only edges whose + // SOURCE is a real node are kept, so a dependent is always a graph node. const nodeIds = new Set(snap.nodes.map((n) => n.id)); const incoming = new Map(); for (const e of snap.links) { @@ -44,14 +44,18 @@ export function renderImpact(snap: GraphSnapshot, pattern: string): string { if (list) list.push(e); else incoming.set(e.target, [e]); } - // BFS over reverse edges. Record the depth + the relation/source that first - // reached each dependent (stable: process queue in id order per level). const depthOf = new Map(); const viaOf = new Map(); - depthOf.set(target.id, 0); - let frontier = [target.id]; + let frontier: string[] = []; + for (const s of seeds) { + if (!depthOf.has(s)) { + depthOf.set(s, 0); + frontier.push(s); + } + } + frontier.sort(); let depth = 0; - while (frontier.length > 0 && depth < MAX_DEPTH) { + while (frontier.length > 0 && depth < maxDepth) { depth++; const next: string[] = []; for (const id of frontier) { @@ -67,6 +71,40 @@ export function renderImpact(snap: GraphSnapshot, pattern: string): string { next.sort(); frontier = next; } + return { depthOf, viaOf }; +} + +/** + * The transitive dependent closure of `seeds` — every node that reaches a + * seed by following edges in reverse, PLUS the seeds themselves. This is the + * blast radius: "if these symbols change, which symbols could be affected?" + * Used by the doc drift detector to widen staleness from changed symbols to + * the docs of their callers. + */ +export function impactedNodes( + snap: GraphSnapshot, + seeds: Iterable, + opts?: { maxDepth?: number }, +): Set { + return new Set(reverseBfs(snap, seeds, opts?.maxDepth).depthOf.keys()); +} + +export function renderImpact(snap: GraphSnapshot, pattern: string): string { + const needle = pattern.toLowerCase(); + const matches = snap.nodes.filter((n) => n.id.toLowerCase().includes(needle)); + if (matches.length === 0) { + return `No node matches "${pattern}". Try cat memory/graph/find/${pattern} to explore.`; + } + if (matches.length > 1) { + const lines = [`"${pattern}" matches ${matches.length} nodes — be more specific:`, ""]; + for (const m of matches.slice(0, 20)) lines.push(` ${m.id}`); + if (matches.length > 20) lines.push(` ... and ${matches.length - 20} more`); + return lines.join("\n"); + } + const target = matches[0]!; + + // Reverse-edge BFS from the single target (shared with impactedNodes). + const { depthOf, viaOf } = reverseBfs(snap, [target.id]); // Collect dependents (everything except the target itself), by depth. const dependents = [...depthOf.entries()].filter(([id]) => id !== target.id); From 266bb3674b40321e75c92a0bcb7e0409da84b646 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:02:26 +0000 Subject: [PATCH 06/39] feat: add doc drift detection via anchors and blast-radius --- src/docs/anchors.ts | 102 ++++++++++++ src/docs/impact.ts | 150 +++++++++++++++++ src/docs/index.ts | 13 ++ tests/shared/docs-impact.test.ts | 271 +++++++++++++++++++++++++++++++ 4 files changed, 536 insertions(+) create mode 100644 src/docs/anchors.ts create mode 100644 src/docs/impact.ts create mode 100644 tests/shared/docs-impact.test.ts diff --git a/src/docs/anchors.ts b/src/docs/anchors.ts new file mode 100644 index 00000000..330d9599 --- /dev/null +++ b/src/docs/anchors.ts @@ -0,0 +1,102 @@ +/** + * Anchors — the join between a doc and the code it describes. + * + * An anchor is `{ symbol_id, content_hash }`: the graph-node id of a symbol + * plus a sha256 fingerprint of that symbol's *source slice* at the moment the + * doc was written. Drift detection is then a pure comparison — re-read the + * symbol's source now, hash it, and if it differs from the stored hash the + * doc is stale. + * + * Why hash the SOURCE SLICE and not the line LOCATION: inserting an import + * above a function shifts its line numbers but not its body. Hashing the + * content (lines start..end joined) is robust to that movement and only + * fires on a real edit to the symbol itself. Line-ending normalization is + * the only normalization applied — we deliberately do NOT strip whitespace + * (indentation is semantic in Python and over-normalizing hides real edits). + */ + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { GraphNode, GraphSnapshot } from "../graph/types.js"; +import type { DocAnchor } from "./read.js"; + +/** Parse a node's `source_location` (`L10`, `L10-20`, or `L10-L20`) → 1-indexed line range. */ +export function parseSourceLocation(loc: string): { startLine: number; endLine: number } | null { + const m = loc.match(/^L(\d+)(?:-L?(\d+))?$/); + if (!m) return null; + const startLine = Number(m[1]); + const endLine = m[2] ? Number(m[2]) : startLine; + if (!Number.isInteger(startLine) || !Number.isInteger(endLine)) return null; + if (startLine < 1 || endLine < startLine) return null; + return { startLine, endLine }; +} + +/** + * Read the source slice for a symbol from the working tree. Returns null if + * the location is unparseable, the file is unreadable, or the line range no + * longer fits the file (a stale location — the symbol moved or shrank). + */ +export function readSymbolSource(node: GraphNode, repoRoot: string): string | null { + const loc = parseSourceLocation(node.source_location); + if (!loc) return null; + let text: string; + try { + text = readFileSync(join(repoRoot, node.source_file), "utf-8"); + } catch { + return null; + } + const lines = text.split(/\r?\n/); + if (loc.endLine > lines.length) return null; + return lines.slice(loc.startLine - 1, loc.endLine).join("\n"); +} + +/** sha256 of a source slice. Line endings are already normalized by the splitter. */ +export function hashSource(src: string): string { + return createHash("sha256").update(src).digest("hex"); +} + +/** Compute the content hash for a symbol, or null if its source can't be read. */ +export function computeSymbolHash(node: GraphNode, repoRoot: string): string | null { + const src = readSymbolSource(node, repoRoot); + return src === null ? null : hashSource(src); +} + +/** Build an anchor for a symbol node, or null if its source can't be read. */ +export function buildAnchor(node: GraphNode, repoRoot: string): DocAnchor | null { + const hash = computeSymbolHash(node, repoRoot); + return hash === null ? null : { symbol_id: node.id, content_hash: hash }; +} + +/** Status of one anchor checked against the current snapshot + working tree. */ +export type AnchorStatus = + | { state: "fresh" } + /** The symbol no longer exists in the graph (removed or renamed). */ + | { state: "missing" } + /** The symbol exists but its source slice can't be read (file/line moved). */ + | { state: "unreadable" } + /** The symbol's source changed since the anchor was taken. */ + | { state: "changed"; from: string; to: string }; + +/** + * Check a single anchor against the current snapshot + working tree. + * `nodeById` is an optional prebuilt index (build once per snapshot when + * checking many anchors — `computeStaleDocs` does this). + */ +export function anchorStatus( + anchor: DocAnchor, + snap: GraphSnapshot, + repoRoot: string, + nodeById?: Map, +): AnchorStatus { + const node = nodeById + ? nodeById.get(anchor.symbol_id) + : snap.nodes.find((n) => n.id === anchor.symbol_id); + if (!node) return { state: "missing" }; + const current = computeSymbolHash(node, repoRoot); + if (current === null) return { state: "unreadable" }; + if (current !== anchor.content_hash) { + return { state: "changed", from: anchor.content_hash, to: current }; + } + return { state: "fresh" }; +} diff --git a/src/docs/impact.ts b/src/docs/impact.ts new file mode 100644 index 00000000..0c457c44 --- /dev/null +++ b/src/docs/impact.ts @@ -0,0 +1,150 @@ +/** + * Doc drift detection — which docs are stale after a code change, and why. + * + * Two complementary signals: + * + * 1. DIRECT (hash): a doc's own anchored symbol changed. Re-read the + * symbol's source slice now, hash it, compare to the stored anchor hash. + * This catches MODIFIED bodies that `diffSnapshots` misses (a symbol + * whose id is unchanged but whose code changed is neither "added" nor + * "removed" in a node-id set diff). + * + * 2. RELATIONAL (blast radius): a doc whose anchored symbol is a transitive + * CALLER of a changed symbol. The caller's own source may be byte-for-byte + * identical (so the hash check passes), yet its doc can be wrong because + * the thing it calls changed shape. We widen from changed symbols to + * their dependents over the graph's reverse edges (`impactedNodes`). + * + * The orchestrator `computeImpactedDocs` unions both. It is the work-list + * generator for the doc-refresh worker (Phase 1 step 5): "these docs, for + * these reasons, need regenerating — and only these." + */ + +import type { GraphSnapshot, GraphNode } from "../graph/types.js"; +import type { SnapshotDiff } from "../graph/diff.js"; +import { impactedNodes } from "../graph/render/impact.js"; +import { computeSymbolHash } from "./anchors.js"; +import type { DocRow } from "./read.js"; + +export type StaleReason = + /** The doc's own anchored symbol's source changed since the anchor was taken. */ + | { kind: "code_changed"; symbol_id: string } + /** The doc's anchored symbol no longer exists in the graph (removed/renamed). */ + | { kind: "symbol_missing"; symbol_id: string } + /** The anchored symbol is a transitive caller of a symbol that changed. */ + | { kind: "caller_changed"; symbol_id: string }; + +export interface ImpactedDoc { + doc_id: string; + reasons: StaleReason[]; +} + +/** + * DIRECT staleness — docs whose own anchored code changed or whose symbol + * vanished. Pure over (snapshot, working tree): does not need a previous + * snapshot, because the anchor hash already encodes "the code as it was". + */ +export function computeStaleDocs(args: { + snap: GraphSnapshot; + docs: DocRow[]; + repoRoot: string; +}): ImpactedDoc[] { + const nodeById = new Map(args.snap.nodes.map((n) => [n.id, n])); + const out: ImpactedDoc[] = []; + for (const doc of args.docs) { + const reasons: StaleReason[] = []; + for (const anchor of doc.anchors) { + const node = nodeById.get(anchor.symbol_id); + if (!node) { + reasons.push({ kind: "symbol_missing", symbol_id: anchor.symbol_id }); + continue; + } + const current = computeSymbolHash(node, args.repoRoot); + if (current === null) { + // Source unreadable at the recorded location — treat as missing so the + // worker re-anchors rather than silently trusting a stale hash. + reasons.push({ kind: "symbol_missing", symbol_id: anchor.symbol_id }); + continue; + } + if (current !== anchor.content_hash) { + reasons.push({ kind: "code_changed", symbol_id: anchor.symbol_id }); + } + } + if (reasons.length > 0) out.push({ doc_id: doc.doc_id, reasons }); + } + return out; +} + +/** + * RELATIONAL widening — docs whose anchored symbol is a transitive caller of + * one of `changedSymbolIds`. Excludes the changed symbols themselves (those + * are already reported by `computeStaleDocs`'s direct hash check), so the two + * passes don't double-attribute the same cause. + */ +export function widenByBlastRadius(args: { + snap: GraphSnapshot; + changedSymbolIds: Iterable; + docs: DocRow[]; +}): ImpactedDoc[] { + const changed = new Set(args.changedSymbolIds); + if (changed.size === 0) return []; + const closure = impactedNodes(args.snap, changed); // dependents + seeds + const out: ImpactedDoc[] = []; + for (const doc of args.docs) { + const reasons: StaleReason[] = []; + for (const anchor of doc.anchors) { + if (closure.has(anchor.symbol_id) && !changed.has(anchor.symbol_id)) { + reasons.push({ kind: "caller_changed", symbol_id: anchor.symbol_id }); + } + } + if (reasons.length > 0) out.push({ doc_id: doc.doc_id, reasons }); + } + return out; +} + +/** + * Full impacted set for a commit delta = DIRECT ∪ RELATIONAL. + * + * The blast-radius is seeded by the union of: + * - structural changes from `diff` (added/removed node ids), if provided, + * - anchored symbols the direct pass found `code_changed`. + * + * Reasons for the same doc_id are merged. The result is deduped by doc_id. + */ +export function computeImpactedDocs(args: { + snap: GraphSnapshot; + docs: DocRow[]; + repoRoot: string; + /** Optional structural diff to seed relational widening with non-anchored changes. */ + diff?: SnapshotDiff | null; +}): ImpactedDoc[] { + const merged = new Map(); + const add = (d: ImpactedDoc): void => { + const cur = merged.get(d.doc_id); + if (cur) cur.push(...d.reasons); + else merged.set(d.doc_id, [...d.reasons]); + }; + + // 1. Direct hash staleness. + const direct = computeStaleDocs({ snap: args.snap, docs: args.docs, repoRoot: args.repoRoot }); + for (const d of direct) add(d); + + // 2. Seed the blast radius: structural diff + directly-changed anchored symbols. + const seeds = new Set(); + if (args.diff) { + for (const n of args.diff.nodes.added) seeds.add(n.id); + for (const n of args.diff.nodes.removed) seeds.add(n.id); + } + for (const d of direct) { + for (const r of d.reasons) { + if (r.kind === "code_changed") seeds.add(r.symbol_id); + } + } + + // 3. Relational widening. + for (const d of widenByBlastRadius({ snap: args.snap, changedSymbolIds: seeds, docs: args.docs })) { + add(d); + } + + return [...merged.entries()].map(([doc_id, reasons]) => ({ doc_id, reasons })); +} diff --git a/src/docs/index.ts b/src/docs/index.ts index 81a64429..a51217c7 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -17,3 +17,16 @@ export type { ListDocsOpts, QueryFn, } from "./read.js"; + +export { + parseSourceLocation, + readSymbolSource, + hashSource, + computeSymbolHash, + buildAnchor, + anchorStatus, +} from "./anchors.js"; +export type { AnchorStatus } from "./anchors.js"; + +export { computeStaleDocs, widenByBlastRadius, computeImpactedDocs } from "./impact.js"; +export type { StaleReason, ImpactedDoc } from "./impact.js"; diff --git a/tests/shared/docs-impact.test.ts b/tests/shared/docs-impact.test.ts new file mode 100644 index 00000000..e2850b28 --- /dev/null +++ b/tests/shared/docs-impact.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; +import { + parseSourceLocation, + readSymbolSource, + computeSymbolHash, + buildAnchor, + anchorStatus, + computeStaleDocs, + widenByBlastRadius, + computeImpactedDocs, + type DocRow, + type DocAnchor, +} from "../../src/docs/index.js"; +import type { GraphNode, GraphSnapshot, GraphEdge } from "../../src/graph/types.js"; + +// ── fixture builders ────────────────────────────────────────────────────────── + +function node(id: string, source_file: string, source_location: string): GraphNode { + const [, name, kind] = id.split(":"); + return { + id, + label: name ?? id, + kind: (kind as GraphNode["kind"]) ?? "function", + source_file, + source_location, + language: "typescript", + exported: true, + }; +} + +function edge(source: string, target: string, relation: GraphEdge["relation"] = "calls"): GraphEdge { + return { source, target, relation, confidence: "EXTRACTED" }; +} + +function snap(nodes: GraphNode[], links: GraphEdge[] = []): GraphSnapshot { + return { nodes, links } as unknown as GraphSnapshot; +} + +function doc(doc_id: string, anchors: DocAnchor[]): DocRow { + return { + id: `row-${doc_id}`, + doc_id, + path: `/docs/p/${doc_id}.md`, + content: "# doc", + anchors, + tier: "fast", + status: "active", + project: "p", + version: 1, + created_at: "2026-06-22T00:00:00Z", + updated_at: "2026-06-22T00:00:00Z", + agent: "manual", + plugin_version: "0", + }; +} + +const sha = (s: string) => createHash("sha256").update(s).digest("hex"); + +// ── parseSourceLocation ───────────────────────────────────────────────────── + +describe("parseSourceLocation", () => { + it("parses single-line and range forms", () => { + expect(parseSourceLocation("L10")).toEqual({ startLine: 10, endLine: 10 }); + expect(parseSourceLocation("L10-20")).toEqual({ startLine: 10, endLine: 20 }); + expect(parseSourceLocation("L10-L20")).toEqual({ startLine: 10, endLine: 20 }); + }); + it("rejects garbage and inverted / zero ranges", () => { + expect(parseSourceLocation("")).toBeNull(); + expect(parseSourceLocation("10")).toBeNull(); + expect(parseSourceLocation("Lx")).toBeNull(); + expect(parseSourceLocation("L0")).toBeNull(); + expect(parseSourceLocation("L20-L10")).toBeNull(); + }); +}); + +// ── source reading + hashing (real files on disk) ─────────────────────────── + +describe("readSymbolSource / computeSymbolHash", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-anchor-")); + // 5-line file; the symbol "foo" is lines 2-3. + writeFileSync(join(dir, "a.ts"), "// header\nfunction foo() {\n return 1;\n}\n// tail\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("reads exactly the symbol's line slice", () => { + const src = readSymbolSource(node("a.ts:foo:function", "a.ts", "L2-L3"), dir); + expect(src).toBe("function foo() {\n return 1;"); + }); + + it("hashes the slice content (robust to lines moving)", () => { + const h1 = computeSymbolHash(node("a.ts:foo:function", "a.ts", "L2-L3"), dir); + expect(h1).toBe(sha("function foo() {\n return 1;")); + // Same body at a different location → SAME hash (location-independent). + writeFileSync(join(dir, "b.ts"), "\n\n\nfunction foo() {\n return 1;\n}\n"); + const h2 = computeSymbolHash(node("a.ts:foo:function", "b.ts", "L4-L5"), dir); + expect(h2).toBe(h1); + }); + + it("returns null for missing file or out-of-range lines", () => { + expect(computeSymbolHash(node("x:foo:function", "missing.ts", "L1"), dir)).toBeNull(); + expect(computeSymbolHash(node("a.ts:foo:function", "a.ts", "L99-L100"), dir)).toBeNull(); + }); + + it("buildAnchor returns {symbol_id, content_hash} or null", () => { + const n = node("a.ts:foo:function", "a.ts", "L2-L3"); + expect(buildAnchor(n, dir)).toEqual({ symbol_id: n.id, content_hash: sha("function foo() {\n return 1;") }); + expect(buildAnchor(node("a.ts:foo:function", "gone.ts", "L1"), dir)).toBeNull(); + }); +}); + +// ── anchorStatus ──────────────────────────────────────────────────────────── + +describe("anchorStatus", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-status-")); + writeFileSync(join(dir, "a.ts"), "function foo() {\n return 1;\n}\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("fresh when the stored hash matches current code", () => { + const n = node("a.ts:foo:function", "a.ts", "L1-L2"); + const a = buildAnchor(n, dir)!; + expect(anchorStatus(a, snap([n]), dir)).toEqual({ state: "fresh" }); + }); + + it("changed when the code differs from the stored hash", () => { + const n = node("a.ts:foo:function", "a.ts", "L1-L2"); + const stale: DocAnchor = { symbol_id: n.id, content_hash: "deadbeef" }; + const st = anchorStatus(stale, snap([n]), dir); + expect(st.state).toBe("changed"); + }); + + it("missing when the symbol is no longer in the graph", () => { + const a: DocAnchor = { symbol_id: "a.ts:gone:function", content_hash: "x" }; + expect(anchorStatus(a, snap([]), dir)).toEqual({ state: "missing" }); + }); + + it("unreadable when the symbol exists but its source can't be read", () => { + const n = node("a.ts:foo:function", "a.ts", "L50-L60"); + const a: DocAnchor = { symbol_id: n.id, content_hash: "x" }; + expect(anchorStatus(a, snap([n]), dir)).toEqual({ state: "unreadable" }); + }); +}); + +// ── computeStaleDocs (direct hash staleness) ──────────────────────────────── + +describe("computeStaleDocs", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-stale-")); + writeFileSync(join(dir, "a.ts"), "function foo() {\n return 1;\n}\n"); + writeFileSync(join(dir, "b.ts"), "function bar() {\n return 2;\n}\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("flags ONLY the doc whose anchored code changed", () => { + const foo = node("a.ts:foo:function", "a.ts", "L1-L2"); + const bar = node("b.ts:bar:function", "b.ts", "L1-L2"); + const freshFoo = buildAnchor(foo, dir)!; // correct hash → fresh + const staleBar: DocAnchor = { symbol_id: bar.id, content_hash: "wrong000" }; // → changed + const result = computeStaleDocs({ + snap: snap([foo, bar]), + docs: [doc("a.ts", [freshFoo]), doc("b.ts", [staleBar])], + repoRoot: dir, + }); + expect(result.map(r => r.doc_id)).toEqual(["b.ts"]); + expect(result[0].reasons).toEqual([{ kind: "code_changed", symbol_id: bar.id }]); + }); + + it("reports symbol_missing when the anchored symbol vanished from the graph", () => { + const result = computeStaleDocs({ + snap: snap([]), + docs: [doc("a.ts", [{ symbol_id: "a.ts:foo:function", content_hash: "x" }])], + repoRoot: dir, + }); + expect(result[0].reasons).toEqual([{ kind: "symbol_missing", symbol_id: "a.ts:foo:function" }]); + }); + + it("returns nothing when every anchor is fresh", () => { + const foo = node("a.ts:foo:function", "a.ts", "L1-L2"); + const result = computeStaleDocs({ + snap: snap([foo]), + docs: [doc("a.ts", [buildAnchor(foo, dir)!])], + repoRoot: dir, + }); + expect(result).toEqual([]); + }); +}); + +// ── widenByBlastRadius (relational staleness) ─────────────────────────────── + +describe("widenByBlastRadius", () => { + // bar() calls foo(); baz() is unrelated. + const foo = node("a.ts:foo:function", "a.ts", "L1"); + const bar = node("b.ts:bar:function", "b.ts", "L1"); + const baz = node("c.ts:baz:function", "c.ts", "L1"); + const s = snap([foo, bar, baz], [edge(bar.id, foo.id, "calls")]); + + it("flags the CALLER's doc when the callee changed, not the unrelated doc", () => { + const result = widenByBlastRadius({ + snap: s, + changedSymbolIds: [foo.id], // foo changed + docs: [ + doc("b.ts", [{ symbol_id: bar.id, content_hash: "h" }]), // bar calls foo → flagged + doc("c.ts", [{ symbol_id: baz.id, content_hash: "h" }]), // baz unrelated → not + ], + }); + expect(result.map(r => r.doc_id)).toEqual(["b.ts"]); + expect(result[0].reasons).toEqual([{ kind: "caller_changed", symbol_id: bar.id }]); + }); + + it("does NOT re-flag the changed symbol's own doc (left to the direct pass)", () => { + const result = widenByBlastRadius({ + snap: s, + changedSymbolIds: [foo.id], + docs: [doc("a.ts", [{ symbol_id: foo.id, content_hash: "h" }])], + }); + expect(result).toEqual([]); + }); + + it("returns nothing when no symbols changed", () => { + expect(widenByBlastRadius({ snap: s, changedSymbolIds: [], docs: [doc("b.ts", [])] })).toEqual([]); + }); +}); + +// ── computeImpactedDocs (direct ∪ relational) ─────────────────────────────── + +describe("computeImpactedDocs", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-impacted-")); + writeFileSync(join(dir, "a.ts"), "function foo() {\n return 1;\n}\n"); + writeFileSync(join(dir, "b.ts"), "function bar() {\n return foo();\n}\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("unions a directly-changed doc with the doc of its caller (one reason each)", () => { + const foo = node("a.ts:foo:function", "a.ts", "L1-L2"); + const bar = node("b.ts:bar:function", "b.ts", "L1-L2"); + const s = snap([foo, bar], [edge(bar.id, foo.id, "calls")]); + // foo's anchor is stale (code_changed); bar's anchor is fresh but bar calls foo. + const staleFoo: DocAnchor = { symbol_id: foo.id, content_hash: "wrong000" }; + const freshBar = buildAnchor(bar, dir)!; + const result = computeImpactedDocs({ + snap: s, + docs: [doc("a.ts", [staleFoo]), doc("b.ts", [freshBar])], + repoRoot: dir, + }); + const byId = Object.fromEntries(result.map(r => [r.doc_id, r.reasons])); + expect(Object.keys(byId).sort()).toEqual(["a.ts", "b.ts"]); + expect(byId["a.ts"]).toEqual([{ kind: "code_changed", symbol_id: foo.id }]); + expect(byId["b.ts"]).toEqual([{ kind: "caller_changed", symbol_id: bar.id }]); + }); + + it("flags nothing when all anchors are fresh and no diff seeds widening", () => { + const foo = node("a.ts:foo:function", "a.ts", "L1-L2"); + const result = computeImpactedDocs({ + snap: snap([foo]), + docs: [doc("a.ts", [buildAnchor(foo, dir)!])], + repoRoot: dir, + }); + expect(result).toEqual([]); + }); +}); From 7e2a1c8b38cac8437eef55c682fda2505f5d0d2f Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:31:20 +0000 Subject: [PATCH 07/39] feat: add doc-edit gate and refresh engine --- src/docs/gate.ts | 105 ++++++++++++++ src/docs/index.ts | 13 ++ src/docs/refresh-llm.ts | 34 +++++ src/docs/refresh.ts | 187 ++++++++++++++++++++++++ tests/shared/docs-refresh.test.ts | 234 ++++++++++++++++++++++++++++++ 5 files changed, 573 insertions(+) create mode 100644 src/docs/gate.ts create mode 100644 src/docs/refresh-llm.ts create mode 100644 src/docs/refresh.ts create mode 100644 tests/shared/docs-refresh.test.ts diff --git a/src/docs/gate.ts b/src/docs/gate.ts new file mode 100644 index 00000000..2cb6140d --- /dev/null +++ b/src/docs/gate.ts @@ -0,0 +1,105 @@ +/** + * The doc-edit gate — objective invariants a proposed doc refresh must pass + * before it is written back. Prose quality has no auto-grader (SkillOpt #6), + * so the gate deliberately checks only things that ARE objectively decidable: + * + * 1. Non-empty, within length cap. + * 2. SLOW tier is human-curated — auto-refresh is rejected outright (the + * protected-section invariant: a fast/automatic edit must never rewrite + * slow, hard-won project knowledge). + * 3. Every anchor in the proposed version references a symbol that still + * exists in the graph — no dangling anchors get persisted. + * 4. Bounded edit (a textual learning rate, SkillOpt): the line-diff between + * the previous and proposed content must stay within budget. A refresh + * that rewrites the whole doc is rejected as slop, not accepted silently. + * + * Pure and synchronous — no I/O, no LLM. Easy to unit-test exhaustively. + */ + +import type { GraphSnapshot } from "../graph/types.js"; +import type { DocAnchor, DocTier } from "./read.js"; + +/** Default ceiling on changed lines for a single bounded refresh. */ +export const DEFAULT_MAX_CHANGED_LINES = 40; +/** Hard content length cap (mirrors the store's own cap). */ +export const GATE_MAX_CONTENT_LENGTH = 50_000; + +export interface GateInput { + tier: DocTier; + /** The current (previous) doc body. Empty string for a brand-new doc. */ + prevContent: string; + /** The proposed new doc body from the refresh worker. */ + newContent: string; + /** The anchors the proposed version would carry. */ + newAnchors: DocAnchor[]; + /** Current graph snapshot — used to confirm anchored symbols still exist. */ + snap: GraphSnapshot; + /** Override the changed-line budget. */ + maxChangedLines?: number; +} + +export interface GateResult { + ok: boolean; + /** Human-readable rejection reasons (empty when ok). */ + reasons: string[]; + /** Number of changed lines measured (for logging / tuning). */ + changedLines: number; +} + +/** + * Count changed lines between two texts via an LCS line diff: + * changed = (prevLines - lcs) + (newLines - lcs) + * This is the classic added+removed edit count — deterministic and stable + * under line reordering noise (unlike a naive set difference). + */ +export function countChangedLines(prev: string, next: string): number { + const a = prev === "" ? [] : prev.split("\n"); + const b = next === "" ? [] : next.split("\n"); + const n = a.length; + const m = b.length; + if (n === 0) return m; + if (m === 0) return n; + // LCS length via rolling DP (O(n*m) time, O(m) space). + let prevRow = new Array(m + 1).fill(0); + for (let i = 1; i <= n; i++) { + const curRow = new Array(m + 1).fill(0); + for (let j = 1; j <= m; j++) { + curRow[j] = a[i - 1] === b[j - 1] + ? prevRow[j - 1] + 1 + : Math.max(prevRow[j], curRow[j - 1]); + } + prevRow = curRow; + } + const lcs = prevRow[m]; + return (n - lcs) + (m - lcs); +} + +/** Run the gate over a proposed doc edit. */ +export function gateDocEdit(input: GateInput): GateResult { + const reasons: string[] = []; + const changedLines = countChangedLines(input.prevContent, input.newContent); + + if (input.newContent.length === 0) { + reasons.push("proposed content is empty"); + } + if (input.newContent.length > GATE_MAX_CONTENT_LENGTH) { + reasons.push(`proposed content exceeds ${GATE_MAX_CONTENT_LENGTH} chars (got ${input.newContent.length})`); + } + if (input.tier === "slow") { + reasons.push("slow-tier docs are human-curated; automatic refresh is not allowed"); + } + + const nodeIds = new Set(input.snap.nodes.map((n) => n.id)); + for (const a of input.newAnchors) { + if (!nodeIds.has(a.symbol_id)) { + reasons.push(`anchor references a symbol absent from the graph: ${a.symbol_id}`); + } + } + + const budget = input.maxChangedLines ?? DEFAULT_MAX_CHANGED_LINES; + if (changedLines > budget) { + reasons.push(`edit exceeds the bounded-change budget: ${changedLines} > ${budget} lines`); + } + + return { ok: reasons.length === 0, reasons, changedLines }; +} diff --git a/src/docs/index.ts b/src/docs/index.ts index a51217c7..ae78e892 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -30,3 +30,16 @@ export type { AnchorStatus } from "./anchors.js"; export { computeStaleDocs, widenByBlastRadius, computeImpactedDocs } from "./impact.js"; export type { StaleReason, ImpactedDoc } from "./impact.js"; + +export { gateDocEdit, countChangedLines, DEFAULT_MAX_CHANGED_LINES, GATE_MAX_CONTENT_LENGTH } from "./gate.js"; +export type { GateInput, GateResult } from "./gate.js"; + +export { refreshDocs, buildRefreshPrompt } from "./refresh.js"; +export type { + GenerateFn, + RefreshContext, + RefreshArgs, + RefreshReport, + RefreshOutcome, + ChangedSymbol, +} from "./refresh.js"; diff --git a/src/docs/refresh-llm.ts b/src/docs/refresh-llm.ts new file mode 100644 index 00000000..c5affffd --- /dev/null +++ b/src/docs/refresh-llm.ts @@ -0,0 +1,34 @@ +/** + * Real host-LLM generator for doc refresh — the production `GenerateFn`. + * + * Reuses the no-API-key seam the wiki worker uses: shell out to the host's + * `claude` CLI via `buildClaudeInvocation` (handles Unix vs Windows `.cmd`). + * The prompt instructs the model to print ONLY the updated markdown, which we + * capture from stdout. + * + * This module is intentionally thin and side-effecting (subprocess); the + * orchestration + gating it feeds (`./refresh.ts`, `./gate.ts`) is pure and + * unit-tested. Per-agent variants (codex/cursor/hermes/pi) mirror the + * wiki-worker forks and can wrap `buildTrailingPromptInvocation` the same way. + */ + +import { execFileSync } from "node:child_process"; +import { buildClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; +import { resolveCliBin } from "../utils/resolve-cli-bin.js"; +import { buildRefreshPrompt, type GenerateFn } from "./refresh.js"; + +/** Build a GenerateFn backed by the host `claude` CLI. */ +export function makeClaudeGenerate(claudeBin?: string, timeoutMs = 120_000): GenerateFn { + const bin = claudeBin ?? resolveCliBin("claude"); + return async (ctx) => { + const prompt = buildRefreshPrompt(ctx); + const inv = buildClaudeInvocation(bin, prompt); + const out = execFileSync(inv.file, inv.args, { + ...inv.options, + encoding: "utf-8", + timeout: timeoutMs, + env: { ...process.env, HIVEMIND_WIKI_WORKER: "1", HIVEMIND_CAPTURE: "false" }, + }); + return (out ?? "").toString().trim(); + }; +} diff --git a/src/docs/refresh.ts b/src/docs/refresh.ts new file mode 100644 index 00000000..cd0396c3 --- /dev/null +++ b/src/docs/refresh.ts @@ -0,0 +1,187 @@ +/** + * Doc refresh orchestrator — turns the drift detector's work-list (step 3) + * into rewritten docs, gated. + * + * Flow per stale doc: + * 1. Re-anchor: recompute each anchor's hash against the current code + * (dropping anchors whose symbol vanished). + * 2. Gather the changed symbols' CURRENT source as context. + * 3. Ask the host LLM (injected `generate`) to produce a bounded rewrite. + * 4. Gate it (objective invariants — see ./gate.ts). + * 5. On pass, `setDoc` a new version with the fresh anchors; on fail, record + * the rejection. Nothing is written when the gate rejects. + * + * The LLM call is INJECTED (`generate`) so this whole module is unit-testable + * with a deterministic stub — no network, no subprocess. The real host-CLI + * implementation lives in ./refresh-llm.ts. + */ + +import { buildAnchor, readSymbolSource } from "./anchors.js"; +import { gateDocEdit, type GateResult } from "./gate.js"; +import { setDoc } from "./write.js"; +import type { DocAnchor, DocRow, QueryFn } from "./read.js"; +import type { ImpactedDoc, StaleReason } from "./impact.js"; +import type { GraphNode, GraphSnapshot } from "../graph/types.js"; + +/** One changed symbol with its current source, handed to the LLM as context. */ +export interface ChangedSymbol { + symbol_id: string; + signature?: string; + source: string; +} + +/** Everything the generator needs to rewrite one doc. */ +export interface RefreshContext { + doc: DocRow; + reasons: StaleReason[]; + changedSymbols: ChangedSymbol[]; +} + +/** Injected LLM call: given context, return the new markdown body. */ +export type GenerateFn = (ctx: RefreshContext) => Promise; + +export interface RefreshOutcome { + doc_id: string; + status: "refreshed" | "rejected" | "skipped"; + /** New version number when refreshed. */ + version?: number; + /** Gate rejection reasons, or skip explanation. */ + reasons?: string[]; +} + +export interface RefreshReport { + outcomes: RefreshOutcome[]; + refreshed: number; + rejected: number; + skipped: number; +} + +export interface RefreshArgs { + query: QueryFn; + tableName: string; + snap: GraphSnapshot; + repoRoot: string; + /** The drift detector's output. */ + impacted: ImpactedDoc[]; + /** Current docs by doc_id (latest versions). */ + docsById: Map; + generate: GenerateFn; + agent?: string; + pluginVersion?: string; + maxChangedLines?: number; +} + +/** Build the prompt for one doc refresh — bounded-edit, freshness-focused. */ +export function buildRefreshPrompt(ctx: RefreshContext): string { + const changed = ctx.changedSymbols + .map((s) => `### ${s.symbol_id}\n${s.signature ? s.signature + "\n" : ""}\n\`\`\`\n${s.source}\n\`\`\``) + .join("\n\n"); + return [ + "You are updating ONE internal documentation file so it matches the code again.", + "The code below changed; the current doc may now be inaccurate.", + "", + "RULES:", + "- Make the SMALLEST edit that restores accuracy. Do NOT rewrite the whole doc.", + "- Preserve the existing structure, headings, and any sections unrelated to the change.", + "- Output ONLY the full updated markdown body. No preamble, no code fences around the whole thing.", + "", + `## Current doc (${ctx.doc.doc_id})`, + ctx.doc.content, + "", + "## Code that changed (current source)", + changed || "(no symbol source available)", + ].join("\n"); +} + +/** + * Recompute anchors against the current snapshot + working tree. Anchors whose + * symbol no longer resolves are dropped (the doc loses that anchor rather than + * carrying a dangling one). + */ +function reanchor(doc: DocRow, nodeById: Map, repoRoot: string): DocAnchor[] { + const out: DocAnchor[] = []; + for (const a of doc.anchors) { + const node = nodeById.get(a.symbol_id); + if (!node) continue; + const fresh = buildAnchor(node, repoRoot); + if (fresh) out.push(fresh); + } + return out; +} + +/** Collect the changed symbols' current source for the prompt context. */ +function gatherChangedSymbols( + reasons: StaleReason[], + nodeById: Map, + repoRoot: string, +): ChangedSymbol[] { + const seen = new Set(); + const out: ChangedSymbol[] = []; + for (const r of reasons) { + if (seen.has(r.symbol_id)) continue; + seen.add(r.symbol_id); + const node = nodeById.get(r.symbol_id); + if (!node) continue; + const source = readSymbolSource(node, repoRoot); + if (source === null) continue; + out.push({ symbol_id: r.symbol_id, signature: node.signature, source }); + } + return out; +} + +/** Refresh every impacted doc, gating each edit. Pure except for `generate` + `query`. */ +export async function refreshDocs(args: RefreshArgs): Promise { + const nodeById = new Map(args.snap.nodes.map((n) => [n.id, n])); + const outcomes: RefreshOutcome[] = []; + + for (const imp of args.impacted) { + const doc = args.docsById.get(imp.doc_id); + if (!doc) { + outcomes.push({ doc_id: imp.doc_id, status: "skipped", reasons: ["no current doc row"] }); + continue; + } + + const newAnchors = reanchor(doc, nodeById, args.repoRoot); + const changedSymbols = gatherChangedSymbols(imp.reasons, nodeById, args.repoRoot); + + let newContent: string; + try { + newContent = await args.generate({ doc, reasons: imp.reasons, changedSymbols }); + } catch (err) { + outcomes.push({ doc_id: imp.doc_id, status: "skipped", reasons: [`generate failed: ${(err as Error).message}`] }); + continue; + } + + const gate: GateResult = gateDocEdit({ + tier: doc.tier, + prevContent: doc.content, + newContent, + newAnchors, + snap: args.snap, + maxChangedLines: args.maxChangedLines, + }); + if (!gate.ok) { + outcomes.push({ doc_id: imp.doc_id, status: "rejected", reasons: gate.reasons }); + continue; + } + + const res = await setDoc(args.query, args.tableName, { + doc_id: doc.doc_id, + path: doc.path, + content: newContent, + anchors: newAnchors, + tier: doc.tier, + project: doc.project, + agent: args.agent ?? "docs-refresh", + plugin_version: args.pluginVersion, + }); + outcomes.push({ doc_id: imp.doc_id, status: "refreshed", version: res.version }); + } + + return { + outcomes, + refreshed: outcomes.filter((o) => o.status === "refreshed").length, + rejected: outcomes.filter((o) => o.status === "rejected").length, + skipped: outcomes.filter((o) => o.status === "skipped").length, + }; +} diff --git a/tests/shared/docs-refresh.test.ts b/tests/shared/docs-refresh.test.ts new file mode 100644 index 00000000..2db48159 --- /dev/null +++ b/tests/shared/docs-refresh.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + gateDocEdit, + countChangedLines, + DEFAULT_MAX_CHANGED_LINES, + GATE_MAX_CONTENT_LENGTH, + buildRefreshPrompt, + refreshDocs, + type DocRow, + type DocAnchor, + type ImpactedDoc, +} from "../../src/docs/index.js"; +import { buildAnchor } from "../../src/docs/index.js"; +import type { GraphNode, GraphSnapshot } from "../../src/graph/types.js"; + +// ── fixtures ──────────────────────────────────────────────────────────────── + +function node(id: string, source_file: string, source_location: string, signature?: string): GraphNode { + return { + id, label: id, kind: "function", source_file, source_location, + language: "typescript", exported: true, signature, + }; +} +function snap(nodes: GraphNode[]): GraphSnapshot { + return { nodes, links: [] } as unknown as GraphSnapshot; +} +function doc(over: Partial = {}): DocRow { + return { + id: "row", doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "old doc", + anchors: [], tier: "fast", status: "active", project: "p", version: 3, + created_at: "t", updated_at: "t", agent: "m", plugin_version: "0", ...over, + }; +} +function mockQuery(script: Array<(sql: string) => unknown>) { + const calls: string[] = []; + let step = 0; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + const out = step < script.length ? script[step++](sql) : []; + return Array.isArray(out) ? (out as Array>) : []; + }); + return { calls, query }; +} + +// ── countChangedLines ───────────────────────────────────────────────────────── + +describe("countChangedLines", () => { + it("is 0 for identical text", () => { + expect(countChangedLines("a\nb\nc", "a\nb\nc")).toBe(0); + }); + it("counts a single changed line as 2 (1 removed + 1 added)", () => { + expect(countChangedLines("a\nb\nc", "a\nX\nc")).toBe(2); + }); + it("counts pure additions and pure removals", () => { + expect(countChangedLines("", "a\nb")).toBe(2); + expect(countChangedLines("a\nb", "")).toBe(2); + }); + it("counts a full rewrite as remove-all + add-all", () => { + expect(countChangedLines("a\nb", "x\ny\nz")).toBe(5); + }); +}); + +// ── gateDocEdit ─────────────────────────────────────────────────────────────── + +describe("gateDocEdit", () => { + const n = node("a.ts:foo:function", "a.ts", "L1"); + const s = snap([n]); + const anchor: DocAnchor = { symbol_id: n.id, content_hash: "h" }; + + it("passes a small edit with valid anchors on a fast doc", () => { + const r = gateDocEdit({ tier: "fast", prevContent: "old doc", newContent: "new doc", newAnchors: [anchor], snap: s }); + expect(r.ok).toBe(true); + expect(r.reasons).toEqual([]); + }); + it("rejects empty content", () => { + const r = gateDocEdit({ tier: "fast", prevContent: "x", newContent: "", newAnchors: [], snap: s }); + expect(r.ok).toBe(false); + expect(r.reasons.join()).toMatch(/empty/); + }); + it("rejects content over the length cap", () => { + const big = "x".repeat(GATE_MAX_CONTENT_LENGTH + 1); + const r = gateDocEdit({ tier: "fast", prevContent: "x", newContent: big, newAnchors: [], snap: s }); + expect(r.ok).toBe(false); + expect(r.reasons.join()).toMatch(/exceeds .* chars/); + }); + it("rejects automatic refresh of a slow-tier doc", () => { + const r = gateDocEdit({ tier: "slow", prevContent: "x", newContent: "y", newAnchors: [anchor], snap: s }); + expect(r.ok).toBe(false); + expect(r.reasons.join()).toMatch(/slow-tier/); + }); + it("rejects an anchor pointing at a symbol absent from the graph", () => { + const r = gateDocEdit({ tier: "fast", prevContent: "x", newContent: "y", newAnchors: [{ symbol_id: "a.ts:gone:function", content_hash: "h" }], snap: s }); + expect(r.ok).toBe(false); + expect(r.reasons.join()).toMatch(/absent from the graph/); + }); + it("rejects an edit beyond the changed-line budget", () => { + const huge = Array.from({ length: DEFAULT_MAX_CHANGED_LINES + 5 }, (_, i) => `line ${i}`).join("\n"); + const r = gateDocEdit({ tier: "fast", prevContent: "old doc", newContent: huge, newAnchors: [anchor], snap: s }); + expect(r.ok).toBe(false); + expect(r.reasons.join()).toMatch(/bounded-change budget/); + }); + it("honors a custom maxChangedLines override", () => { + const r = gateDocEdit({ tier: "fast", prevContent: "a", newContent: "b\nc\nd", newAnchors: [anchor], snap: s, maxChangedLines: 1 }); + expect(r.ok).toBe(false); + }); +}); + +// ── buildRefreshPrompt ──────────────────────────────────────────────────────── + +describe("buildRefreshPrompt", () => { + it("embeds the current doc, the changed symbol source, and bounded-edit rules", () => { + const p = buildRefreshPrompt({ + doc: doc({ content: "# My Doc\nbody" }), + reasons: [{ kind: "code_changed", symbol_id: "a.ts:foo:function" }], + changedSymbols: [{ symbol_id: "a.ts:foo:function", signature: "function foo(): number", source: "function foo() { return 42; }" }], + }); + expect(p).toContain("SMALLEST edit"); + expect(p).toContain("# My Doc"); + expect(p).toContain("function foo() { return 42; }"); + expect(p).toContain("a.ts:foo:function"); + }); + it("degrades gracefully when there is no symbol source", () => { + const p = buildRefreshPrompt({ doc: doc(), reasons: [], changedSymbols: [] }); + expect(p).toContain("(no symbol source available)"); + }); +}); + +// ── refreshDocs (orchestrator) ──────────────────────────────────────────────── + +describe("refreshDocs", () => { + let dir: string; + let foo: GraphNode; + let s: GraphSnapshot; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-refresh-")); + writeFileSync(join(dir, "a.ts"), "export function foo() {\n return 1;\n}\n"); + foo = node("a.ts:foo:function", "a.ts", "L1-L3", "function foo(): number"); + s = snap([foo]); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + const impacted = (): ImpactedDoc[] => [{ doc_id: "a.ts", reasons: [{ kind: "code_changed", symbol_id: foo.id }] }]; + + it("refreshes a stale doc: re-anchors, gates, and setDoc version-bumps", async () => { + const d = doc({ anchors: [{ symbol_id: foo.id, content_hash: "stale" }] }); + const { calls, query } = mockQuery([ + () => [{ id: "r", doc_id: "a.ts", version: 3, content: "old doc", anchors: "[]", tier: "fast", status: "active", project: "p", created_at: "t", updated_at: "t" }], // getDocLatest + () => [], // INSERT + ]); + const generate = vi.fn(async () => "new doc body"); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map([["a.ts", d]]), generate, + }); + expect(report.refreshed).toBe(1); + expect(report.outcomes[0]).toMatchObject({ doc_id: "a.ts", status: "refreshed", version: 4 }); + expect(generate).toHaveBeenCalledOnce(); + // 2 queries: getDocLatest + INSERT. The INSERT carries the FRESH anchor + // (recomputed from current code), not the stale stored hash. + expect(calls).toHaveLength(2); + expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + expect(calls[1]).toContain("new doc body"); + expect(calls[1]).toContain(buildAnchor(foo, dir)!.content_hash); + expect(calls[1]).not.toContain("stale"); + }); + + it("rejects an over-budget rewrite — no write happens", async () => { + const d = doc({ anchors: [{ symbol_id: foo.id, content_hash: "stale" }] }); + const { calls, query } = mockQuery([]); + const huge = Array.from({ length: DEFAULT_MAX_CHANGED_LINES + 10 }, (_, i) => `l${i}`).join("\n"); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map([["a.ts", d]]), generate: async () => huge, + }); + expect(report.rejected).toBe(1); + expect(report.outcomes[0].status).toBe("rejected"); + expect(report.outcomes[0].reasons?.join()).toMatch(/budget/); + expect(calls).toHaveLength(0); // nothing written + }); + + it("rejects a slow-tier doc outright", async () => { + const d = doc({ tier: "slow", anchors: [{ symbol_id: foo.id, content_hash: "x" }] }); + const { calls, query } = mockQuery([]); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map([["a.ts", d]]), generate: async () => "small", + }); + expect(report.rejected).toBe(1); + expect(calls).toHaveLength(0); + }); + + it("skips an impacted doc that has no current row", async () => { + const { query } = mockQuery([]); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map(), generate: async () => "x", + }); + expect(report.skipped).toBe(1); + expect(report.outcomes[0].status).toBe("skipped"); + }); + + it("skips when the generator throws", async () => { + const d = doc({ anchors: [{ symbol_id: foo.id, content_hash: "x" }] }); + const { calls, query } = mockQuery([]); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map([["a.ts", d]]), + generate: async () => { throw new Error("LLM down"); }, + }); + expect(report.skipped).toBe(1); + expect(report.outcomes[0].reasons?.join()).toMatch(/LLM down/); + expect(calls).toHaveLength(0); + }); + + it("drops a dangling anchor when its symbol vanished from the graph", async () => { + // doc anchored to foo + a gone symbol; snapshot only has foo. + const d = doc({ anchors: [{ symbol_id: foo.id, content_hash: "x" }, { symbol_id: "a.ts:gone:function", content_hash: "y" }] }); + const { calls, query } = mockQuery([ + () => [{ id: "r", doc_id: "a.ts", version: 1, content: "old", anchors: "[]", tier: "fast", status: "active", project: "p", created_at: "t", updated_at: "t" }], + () => [], + ]); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: s, repoRoot: dir, + impacted: impacted(), docsById: new Map([["a.ts", d]]), generate: async () => "small", + }); + expect(report.refreshed).toBe(1); + // The INSERT must carry only foo's anchor, not the gone one. + expect(calls[1]).toContain(foo.id); + expect(calls[1]).not.toContain("a.ts:gone:function"); + }); +}); From 68a141f29f69bec40c822338747fd4ee1023f743 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:31:25 +0000 Subject: [PATCH 08/39] feat: add loadCurrentSnapshot for non-vfs callers --- src/graph/load-current.ts | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/graph/load-current.ts diff --git a/src/graph/load-current.ts b/src/graph/load-current.ts new file mode 100644 index 00000000..64487a86 --- /dev/null +++ b/src/graph/load-current.ts @@ -0,0 +1,45 @@ +/** + * Load the current local graph snapshot for a working directory. + * + * Mirrors the resolution `vfs-handler.ts` uses (derive repo key → repo dir → + * last build for this worktree → read the snapshot json), extracted as a + * standalone loader so non-VFS callers (the docs refresh command) can get the + * snapshot without going through the VFS rendering layer. Returns null when + * no graph has been built for this worktree, or the snapshot is missing / + * malformed — callers print a "run `hivemind graph build` first" message. + */ + +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { deriveProjectKey } from "../utils/repo-identity.js"; +import { readLastBuild } from "./last-build.js"; +import { repoDir } from "./snapshot.js"; +import type { GraphSnapshot } from "./types.js"; + +/** Stable per-worktree id — same derivation the VFS handler uses. */ +export function workTreeIdFor(cwd: string): string { + return createHash("sha256").update(cwd).digest("hex").slice(0, 16); +} + +/** Load the latest built snapshot for `cwd`, or null if unavailable/invalid. */ +export function loadCurrentSnapshot(cwd: string): GraphSnapshot | null { + let baseDir: string; + try { + baseDir = repoDir(deriveProjectKey(cwd).key); + } catch { + return null; + } + const last = readLastBuild(baseDir, workTreeIdFor(cwd)); + if (last === null) return null; + const fileBase = last.commit_sha ?? last.snapshot_sha256; + const snapPath = join(baseDir, "snapshots", `${fileBase}.json`); + if (!existsSync(snapPath)) return null; + try { + const snap = JSON.parse(readFileSync(snapPath, "utf8")) as GraphSnapshot; + if (!Array.isArray(snap.nodes) || !Array.isArray(snap.links)) return null; + return snap; + } catch { + return null; + } +} From 71c8b0d5dc949013aea5f55796ff6471c07c8215 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:31:30 +0000 Subject: [PATCH 09/39] feat: add docs refresh command and anchor authoring --- src/commands/docs.ts | 106 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 6f1fd594..d4b6472c 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -32,9 +32,15 @@ import { archiveDoc, listDocs, getDocLatest, + computeImpactedDocs, + refreshDocs, + buildAnchor, type DocRow, type DocTier, + type DocAnchor, } from "../docs/index.js"; +import { makeClaudeGenerate } from "../docs/refresh-llm.js"; +import { loadCurrentSnapshot } from "../graph/load-current.js"; import { isMissingTableError } from "../deeplake-schema.js"; const USAGE = ` @@ -45,6 +51,10 @@ Usage: hivemind docs show hivemind docs list [--project P] [--status active|archived|all] [--limit N] hivemind docs archive + hivemind docs refresh [--cwd ] [--dry-run] + Detect docs whose anchored code drifted (vs the current graph) and + regenerate them via the host LLM, gated. --dry-run only reports the + impacted docs without calling the LLM or writing anything. `.trim(); function requireConfig(): NonNullable> { @@ -67,6 +77,21 @@ function flagValue(args: string[], name: string): string | undefined { return args[idx].includes("=") ? args[idx].split("=", 2)[1] : args[idx + 1]; } +/** Collect ALL values of a repeatable flag (e.g. --anchor X --anchor Y). */ +function flagValues(args: string[], name: string): string[] { + const out: string[] = []; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === name) { + if (args[i + 1] !== undefined) out.push(args[i + 1]); + i++; + } else if (a.startsWith(`${name}=`)) { + out.push(a.split("=", 2)[1]); + } + } + return out; +} + function parseStatus(args: string[]): "active" | "archived" | "all" { const raw = flagValue(args, "--status"); if (raw === undefined) return "active"; @@ -97,7 +122,7 @@ function parseLimit(args: string[]): number { return n; } -const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit"]); +const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor"]); /** Drop flag tokens (and their values) so positional scan sees only doc-id / content. */ function stripKnownFlags(args: string[]): string[] { @@ -149,7 +174,7 @@ export async function runDocsCommand(args: string[]): Promise { // Only write subcommands need DDL — read-only show/list fall back to // isMissingTableError so a fresh-install user doesn't pay a CREATE round-trip. - const WRITE_SUBS = new Set(["set", "archive"]); + const WRITE_SUBS = new Set(["set", "archive", "refresh"]); if (WRITE_SUBS.has(sub)) { await api.ensureDocsTable(tableName); } @@ -166,11 +191,41 @@ export async function runDocsCommand(args: string[]): Promise { const content = resolveContent(positional[1], args); const path = flagValue(args, "--path") ?? defaultVfsPath(project, docId); const tier = parseTier(args); + // Optional anchors: build from the current graph so the doc is tied to the + // code it describes (enables drift detection by `docs refresh`). + const anchorIds = flagValues(args, "--anchor"); + let anchors: DocAnchor[] | undefined; + if (anchorIds.length > 0) { + const snap = loadCurrentSnapshot(flagValue(args, "--cwd") ?? process.cwd()); + if (!snap) { + console.error("--anchor needs a built graph. Run `hivemind graph build` first."); + process.exit(1); + throw new Error("unreachable"); + } + const nodeById = new Map(snap.nodes.map((n) => [n.id, n])); + anchors = []; + for (const sid of anchorIds) { + const node = nodeById.get(sid); + if (!node) { + console.error(`--anchor: symbol not in graph: ${sid}`); + process.exit(1); + throw new Error("unreachable"); + } + const a = buildAnchor(node, flagValue(args, "--cwd") ?? process.cwd()); + if (!a) { + console.error(`--anchor: could not read source for ${sid}`); + process.exit(1); + throw new Error("unreachable"); + } + anchors.push(a); + } + } try { const out = await setDoc(query, tableName, { doc_id: docId, path, content, + anchors, tier, project, agent: cfg.userName, @@ -250,6 +305,53 @@ export async function runDocsCommand(args: string[]): Promise { return; } + if (sub === "refresh") { + const cwd = flagValue(args, "--cwd") ?? process.cwd(); + const dryRun = args.includes("--dry-run"); + const snap = loadCurrentSnapshot(cwd); + if (!snap) { + console.error("No local graph for this directory. Run `hivemind graph build` first."); + process.exit(1); + throw new Error("unreachable"); + } + let docs: DocRow[] = []; + try { + docs = await listDocs(query, tableName, { status: "active", limit: 100000 }); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + const impacted = computeImpactedDocs({ snap, docs, repoRoot: cwd }); + if (impacted.length === 0) { + console.log("(no docs need refreshing — all anchors fresh)"); + return; + } + if (dryRun) { + console.log(`${impacted.length} doc(s) would be refreshed:`); + for (const i of impacted) { + console.log(` ${i.doc_id} [${i.reasons.map((r) => r.kind).join(", ")}]`); + } + return; + } + const docsById = new Map(docs.map((d) => [d.doc_id, d])); + const report = await refreshDocs({ + query, + tableName, + snap, + repoRoot: cwd, + impacted, + docsById, + generate: makeClaudeGenerate(), + agent: cfg.userName, + pluginVersion, + }); + console.log(`Refreshed ${report.refreshed}, rejected ${report.rejected}, skipped ${report.skipped}.`); + for (const o of report.outcomes) { + if (o.status === "refreshed") console.log(` refreshed ${o.doc_id} → v${o.version}`); + else console.log(` ${o.status} ${o.doc_id}: ${(o.reasons ?? []).join("; ")}`); + } + return; + } + console.error(`Unknown subcommand: ${sub}`); console.error(USAGE); process.exit(1); From 2ddbcd6987a3b2f999d94829f30d30b9beb3641b Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:31:35 +0000 Subject: [PATCH 10/39] feat: trigger doc refresh from graph build opt-in --- src/commands/graph.ts | 8 ++++ src/docs/auto-refresh-trigger.ts | 38 +++++++++++++++++++ .../shared/docs-auto-refresh-trigger.test.ts | 31 +++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 src/docs/auto-refresh-trigger.ts create mode 100644 tests/shared/docs-auto-refresh-trigger.test.ts diff --git a/src/commands/graph.ts b/src/commands/graph.ts index 31949e3d..9a9ddbea 100644 --- a/src/commands/graph.ts +++ b/src/commands/graph.ts @@ -43,6 +43,7 @@ import type { GraphObservation, } from "../graph/types.js"; import { deriveProjectKey } from "../utils/repo-identity.js"; +import { maybeSpawnDocsRefresh } from "../docs/auto-refresh-trigger.js"; const USAGE = `hivemind graph — codebase-graph commands (Phase 1.5) @@ -537,6 +538,13 @@ export async function runBuildCommand(args: string[]): Promise { console.warn(`Cloud: push error (non-fatal): ${pushOutcome.message}`); break; } + + // Step 8: after the snapshot is fresh on disk, optionally refresh any docs + // whose anchored code drifted. Opt-in (HIVEMIND_DOCS_AUTO_REFRESH=1), + // detached, best-effort — never blocks the build. + if (maybeSpawnDocsRefresh(cwd)) { + console.log("Docs: spawned drift refresh (HIVEMIND_DOCS_AUTO_REFRESH=1)"); + } } // ─── pull subcommand ─────────────────────────────────────────────────────── diff --git a/src/docs/auto-refresh-trigger.ts b/src/docs/auto-refresh-trigger.ts new file mode 100644 index 00000000..11510d63 --- /dev/null +++ b/src/docs/auto-refresh-trigger.ts @@ -0,0 +1,38 @@ +/** + * Step 8 trigger — kick a doc refresh after a graph build, opt-in. + * + * `hivemind graph build` runs from the post-commit hook; the instant it + * finishes, the snapshot on disk is fresh, so this is the race-free place to + * detect drift and regenerate docs. We do NOT touch the hook body itself + * (shipped + tested) — we hang the trigger off the end of the build command. + * + * Gated behind `HIVEMIND_DOCS_AUTO_REFRESH=1` and OFF by default: auto-refresh + * shells out to the host LLM on every commit, which costs tokens and time, so + * users opt in explicitly (mirrors `HIVEMIND_AUTO_KPI_FROM_COMMITS`). The + * refresh runs in a detached child so the build never blocks on it. + */ + +import { spawnDetachedNodeWorker } from "../utils/spawn-detached.js"; + +export interface AutoRefreshDeps { + env?: NodeJS.ProcessEnv; + /** The CLI entry script to re-invoke (defaults to the running cli bundle). */ + cliEntry?: string; + /** Injectable spawn for tests. */ + spawn?: (workerPath: string, args: readonly string[]) => void; +} + +/** + * Spawn `hivemind docs refresh --cwd ` detached when auto-refresh is + * enabled. Returns true if a refresh was spawned, false if it was a no-op + * (flag off, or no CLI entry to re-invoke). Best-effort and non-throwing. + */ +export function maybeSpawnDocsRefresh(cwd: string, deps: AutoRefreshDeps = {}): boolean { + const env = deps.env ?? process.env; + if (env.HIVEMIND_DOCS_AUTO_REFRESH !== "1") return false; + const cliEntry = deps.cliEntry ?? process.argv[1]; + if (!cliEntry) return false; + const spawn = deps.spawn ?? spawnDetachedNodeWorker; + spawn(cliEntry, ["docs", "refresh", "--cwd", cwd]); + return true; +} diff --git a/tests/shared/docs-auto-refresh-trigger.test.ts b/tests/shared/docs-auto-refresh-trigger.test.ts new file mode 100644 index 00000000..781bb9ef --- /dev/null +++ b/tests/shared/docs-auto-refresh-trigger.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from "vitest"; +import { maybeSpawnDocsRefresh } from "../../src/docs/auto-refresh-trigger.js"; + +describe("maybeSpawnDocsRefresh", () => { + it("no-ops and returns false when the flag is unset", () => { + const spawn = vi.fn(); + const fired = maybeSpawnDocsRefresh("/repo", { env: {}, cliEntry: "/cli.js", spawn }); + expect(fired).toBe(false); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("no-ops when the flag is set to anything other than '1'", () => { + const spawn = vi.fn(); + expect(maybeSpawnDocsRefresh("/repo", { env: { HIVEMIND_DOCS_AUTO_REFRESH: "true" }, cliEntry: "/cli.js", spawn })).toBe(false); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("spawns `docs refresh --cwd ` detached when the flag is '1'", () => { + const spawn = vi.fn(); + const fired = maybeSpawnDocsRefresh("/repo/here", { env: { HIVEMIND_DOCS_AUTO_REFRESH: "1" }, cliEntry: "/path/cli.js", spawn }); + expect(fired).toBe(true); + expect(spawn).toHaveBeenCalledWith("/path/cli.js", ["docs", "refresh", "--cwd", "/repo/here"]); + }); + + it("returns false when there is no CLI entry to re-invoke", () => { + const spawn = vi.fn(); + const fired = maybeSpawnDocsRefresh("/repo", { env: { HIVEMIND_DOCS_AUTO_REFRESH: "1" }, cliEntry: "", spawn }); + expect(fired).toBe(false); + expect(spawn).not.toHaveBeenCalled(); + }); +}); From 7822f38cac5dceb2f5732b1c43528e4c9cb0a62b Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:48:41 +0000 Subject: [PATCH 11/39] test: cover docs refresh engine, cli, and snapshot loader --- tests/claude-code/cli-docs.test.ts | 223 ++++++++++++++++++++++++ tests/shared/docs-refresh-llm.test.ts | 51 ++++++ tests/shared/graph/load-current.test.ts | 74 ++++++++ 3 files changed, 348 insertions(+) create mode 100644 tests/claude-code/cli-docs.test.ts create mode 100644 tests/shared/docs-refresh-llm.test.ts create mode 100644 tests/shared/graph/load-current.test.ts diff --git a/tests/claude-code/cli-docs.test.ts b/tests/claude-code/cli-docs.test.ts new file mode 100644 index 00000000..94b1b03f --- /dev/null +++ b/tests/claude-code/cli-docs.test.ts @@ -0,0 +1,223 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +/** + * CLI handler tests for `hivemind docs`. The handler is thin (argparse + + * dispatch into the real src/docs store), so we mock the network boundary + * (`DeeplakeApi`), config, version, and the two heavy collaborators the + * refresh path would otherwise reach (the graph snapshot loader and the host + * LLM). The store functions run for real against the query spy. + */ + +const ensureDocsTableMock = vi.fn(); +const queryMock = vi.fn(); +const loadCurrentSnapshotMock = vi.fn(); + +vi.mock("../../src/config.js", () => ({ loadConfig: vi.fn() })); +vi.mock("../../src/deeplake-api.js", () => ({ + DeeplakeApi: class { + constructor() { /* nothing */ } + ensureDocsTable(name: string) { return ensureDocsTableMock(name); } + query(sql: string) { return queryMock(sql); } + }, +})); +vi.mock("../../src/cli/version.js", () => ({ getVersion: () => "0.7.99" })); +vi.mock("../../src/graph/load-current.js", () => ({ + loadCurrentSnapshot: (...a: unknown[]) => loadCurrentSnapshotMock(...a), +})); +vi.mock("../../src/docs/refresh-llm.js", () => ({ + makeClaudeGenerate: () => async () => "stub", +})); + +import { runDocsCommand } from "../../src/commands/docs.js"; +import { loadConfig } from "../../src/config.js"; +const loadConfigMock = loadConfig as unknown as ReturnType; + +const VALID_CONFIG = { + token: "tok", orgId: "org", orgName: "OrgName", userName: "alice@activeloop.ai", + workspaceId: "ws", apiUrl: "https://api", tableName: "memory", + sessionsTableName: "sessions", skillsTableName: "skills", rulesTableName: "hivemind_rules", + goalsTableName: "g", kpisTableName: "k", docsTableName: "hivemind_docs", + codebaseTableName: "codebase", memoryPath: "/tmp/mem", +}; + +function docRow(over: Record = {}) { + return { + id: "row", doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "body", + anchors: "[]", tier: "fast", status: "active", project: "p", version: 1, + created_at: "t", updated_at: "t", agent: "m", plugin_version: "0", ...over, + }; +} + +let logged: string[] = []; +let erred: string[] = []; +let logSpy: ReturnType; +let errSpy: ReturnType; +let exitSpy: ReturnType; + +async function run(args: string[]): Promise { + try { + await runDocsCommand(args); + return null; + } catch (e) { + const m = /__EXIT_(\d+)__/.exec((e as Error).message); + if (m) return Number(m[1]); + throw e; + } +} + +beforeEach(() => { + logged = []; erred = []; + ensureDocsTableMock.mockReset().mockResolvedValue(undefined); + queryMock.mockReset().mockResolvedValue([]); + loadCurrentSnapshotMock.mockReset().mockReturnValue({ nodes: [], links: [] }); + loadConfigMock.mockReset().mockReturnValue(VALID_CONFIG); + logSpy = vi.spyOn(console, "log").mockImplementation((...a: any[]) => { logged.push(a.join(" ")); }); + errSpy = vi.spyOn(console, "error").mockImplementation((...a: any[]) => { erred.push(a.join(" ")); }); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`__EXIT_${code ?? 0}__`); + }) as any); +}); +afterEach(() => { + logSpy.mockRestore(); errSpy.mockRestore(); exitSpy.mockRestore(); +}); + +describe("hivemind docs — dispatch & usage", () => { + it("prints usage with no subcommand", async () => { + await run([]); + expect(logged.join("\n")).toMatch(/hivemind docs/); + }); + it("exits 2 when not logged in", async () => { + loadConfigMock.mockReturnValue(null); + expect(await run(["list"])).toBe(2); + expect(erred.join()).toMatch(/Not logged in/); + }); + it("errors on an unknown subcommand", async () => { + expect(await run(["frobnicate"])).toBe(1); + expect(erred.join()).toMatch(/Unknown subcommand/); + }); +}); + +describe("hivemind docs set", () => { + it("requires a doc-id", async () => { + expect(await run(["set"])).toBe(1); + expect(erred.join()).toMatch(/Missing doc-id/); + }); + it("creates v1 via the real store (SELECT then INSERT)", async () => { + await run(["set", "a.ts", "the body", "--project", "p"]); + expect(ensureDocsTableMock).toHaveBeenCalledWith("hivemind_docs"); + const sqls = queryMock.mock.calls.map((c) => c[0] as string); + expect(sqls.some((s) => /^SELECT/.test(s))).toBe(true); + expect(sqls.some((s) => /^INSERT INTO "hivemind_docs"/.test(s))).toBe(true); + expect(logged.join()).toMatch(/Set doc a\.ts → v1/); + }); + it("rejects --anchor when the symbol is not in the graph", async () => { + expect(await run(["set", "a.ts", "body", "--anchor", "a.ts:missing:function"])).toBe(1); + expect(erred.join()).toMatch(/symbol not in graph/); + }); +}); + +describe("hivemind docs show", () => { + it("prints (no doc) when nothing matches", async () => { + await run(["show", "a.ts"]); + expect(logged.join()).toMatch(/\(no doc for a\.ts\)/); + }); + it("prints metadata and body when found", async () => { + queryMock.mockResolvedValueOnce([docRow({ version: 3, content: "hello world" })]); + await run(["show", "a.ts"]); + expect(logged.join("\n")).toMatch(/version: 3/); + expect(logged.join("\n")).toMatch(/hello world/); + }); + it("requires a doc-id", async () => { + expect(await run(["show"])).toBe(1); + }); +}); + +describe("hivemind docs list", () => { + it("reports empty", async () => { + await run(["list"]); + expect(logged.join()).toMatch(/no docs with status=active/); + }); + it("lists rows", async () => { + queryMock.mockResolvedValueOnce([docRow({ doc_id: "x.ts", version: 2 })]); + await run(["list", "--status", "all"]); + expect(logged.join("\n")).toMatch(/x\.ts/); + }); + it("rejects a bad --status", async () => { + expect(await run(["list", "--status", "bogus"])).toBe(1); + }); +}); + +describe("hivemind docs archive", () => { + it("requires a doc-id", async () => { + expect(await run(["archive"])).toBe(1); + }); + it("archives an existing doc (version bump)", async () => { + queryMock.mockResolvedValueOnce([docRow({ version: 2 })]); // getDocLatest + await run(["archive", "a.ts"]); + const sqls = queryMock.mock.calls.map((c) => c[0] as string); + expect(sqls.some((s) => /INSERT INTO "hivemind_docs"/.test(s) && /archived/.test(s))).toBe(true); + expect(logged.join()).toMatch(/Archived doc a\.ts → v3/); + }); +}); + +describe("hivemind docs refresh", () => { + it("errors when no local graph exists", async () => { + loadCurrentSnapshotMock.mockReturnValue(null); + expect(await run(["refresh"])).toBe(1); + expect(erred.join()).toMatch(/No local graph/); + }); + it("reports when nothing needs refreshing", async () => { + // empty graph + no docs → no impact, no LLM call + await run(["refresh", "--dry-run"]); + expect(logged.join()).toMatch(/no docs need refreshing/); + }); +}); + +describe("hivemind docs — anchored authoring + refresh over real files", () => { + let dir: string; + const fooNode = { + id: "f.ts:foo:function", label: "foo", kind: "function", + source_file: "f.ts", source_location: "L1-L3", language: "typescript", + exported: true, signature: "function foo(): number", + }; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "cli-docs-")); + writeFileSync(join(dir, "f.ts"), "export function foo() {\n return 1;\n}\n"); + loadCurrentSnapshotMock.mockReturnValue({ nodes: [fooNode], links: [] }); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("set --anchor builds an anchor from the graph and stores it", async () => { + await run(["set", "f.ts", "doc body", "--anchor", "f.ts:foo:function", "--cwd", dir, "--project", "p"]); + const insert = queryMock.mock.calls.map((c) => c[0] as string).find((s) => /INSERT INTO "hivemind_docs"/.test(s))!; + expect(insert).toBeDefined(); + // the anchors column carries the symbol id (not the empty [] default) + expect(insert).toContain("f.ts:foo:function"); + expect(insert).toContain("content_hash"); + expect(logged.join()).toMatch(/Set doc f\.ts → v1/); + }); + + it("refresh regenerates an impacted doc through the stub generator + gate", async () => { + // doc anchored to foo with a STALE hash → drift detected → refreshed. + const stale = JSON.stringify([{ symbol_id: "f.ts:foo:function", content_hash: "stale000" }]); + queryMock.mockResolvedValue([docRow({ doc_id: "f.ts", content: "old body", anchors: stale })]); + await run(["refresh", "--cwd", dir]); + expect(logged.join("\n")).toMatch(/Refreshed 1/); + const insert = queryMock.mock.calls.map((c) => c[0] as string).find((s) => /INSERT INTO "hivemind_docs"/.test(s)); + expect(insert).toBeDefined(); + expect(insert!).toContain("stub"); // the generated body landed + }); + + it("refresh prints a rejection outcome when the gate rejects (slow tier)", async () => { + const stale = JSON.stringify([{ symbol_id: "f.ts:foo:function", content_hash: "stale000" }]); + queryMock.mockResolvedValue([docRow({ doc_id: "f.ts", content: "old body", anchors: stale, tier: "slow" })]); + await run(["refresh", "--cwd", dir]); + expect(logged.join("\n")).toMatch(/rejected f\.ts:.*slow-tier/); + // no write happened — slow docs are not auto-refreshed + const inserts = queryMock.mock.calls.map((c) => c[0] as string).filter((s) => /INSERT/.test(s)); + expect(inserts).toHaveLength(0); + }); +}); diff --git a/tests/shared/docs-refresh-llm.test.ts b/tests/shared/docs-refresh-llm.test.ts new file mode 100644 index 00000000..27222444 --- /dev/null +++ b/tests/shared/docs-refresh-llm.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; + +// Mock the subprocess boundary so the test never shells out to a real CLI. +const execFileSyncMock = vi.fn(); +vi.mock("node:child_process", () => ({ + execFileSync: (...args: unknown[]) => execFileSyncMock(...args), +})); + +import { makeClaudeGenerate } from "../../src/docs/refresh-llm.js"; +import type { RefreshContext } from "../../src/docs/index.js"; + +const ctx: RefreshContext = { + doc: { + id: "r", doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "old doc", + anchors: [], tier: "fast", status: "active", project: "p", version: 1, + created_at: "t", updated_at: "t", agent: "m", plugin_version: "0", + }, + reasons: [{ kind: "code_changed", symbol_id: "a.ts:foo:function" }], + changedSymbols: [{ symbol_id: "a.ts:foo:function", signature: "function foo(): number", source: "function foo() { return 42; }" }], +}; + +beforeEach(() => execFileSyncMock.mockReset()); + +describe("makeClaudeGenerate", () => { + it("returns the model's stdout, trimmed", async () => { + execFileSyncMock.mockReturnValue(" # Updated doc\nbody\n\n"); + const gen = makeClaudeGenerate("/usr/bin/claude"); + const out = await gen(ctx); + expect(out).toBe("# Updated doc\nbody"); + }); + + it("passes the refresh prompt to the claude binary", async () => { + execFileSyncMock.mockReturnValue("x"); + const gen = makeClaudeGenerate("/usr/bin/claude"); + await gen(ctx); + expect(execFileSyncMock).toHaveBeenCalledOnce(); + const [file, args] = execFileSyncMock.mock.calls[0] as [string, string[]]; + expect(file).toBe("/usr/bin/claude"); + // On unix buildClaudeInvocation puts the prompt as a positional arg; it must + // carry the changed symbol source so the model can update the doc. + const joined = args.join(" "); + expect(joined).toContain("function foo() { return 42; }"); + expect(joined).toContain("SMALLEST edit"); + }); + + it("tolerates empty/undefined stdout", async () => { + execFileSyncMock.mockReturnValue(undefined); + const gen = makeClaudeGenerate("/usr/bin/claude"); + expect(await gen(ctx)).toBe(""); + }); +}); diff --git a/tests/shared/graph/load-current.test.ts b/tests/shared/graph/load-current.test.ts new file mode 100644 index 00000000..4e467446 --- /dev/null +++ b/tests/shared/graph/load-current.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadCurrentSnapshot, workTreeIdFor } from "../../../src/graph/load-current.js"; +import { repoDir } from "../../../src/graph/snapshot.js"; +import { deriveProjectKey } from "../../../src/utils/repo-identity.js"; + +const ORIG = process.env.HIVEMIND_GRAPHS_HOME; +let graphsHome: string; +let cwd: string; + +function baseDirFor(dir: string): string { + return repoDir(deriveProjectKey(dir).key); +} + +function writeLastBuild(dir: string, state: Record): void { + const p = join(baseDirFor(dir), "worktrees", workTreeIdFor(dir), ".last-build.json"); + mkdirSync(join(p, ".."), { recursive: true }); + writeFileSync(p, JSON.stringify(state)); +} + +function writeSnapshotFile(dir: string, fileBase: string, body: string): void { + const snapDir = join(baseDirFor(dir), "snapshots"); + mkdirSync(snapDir, { recursive: true }); + writeFileSync(join(snapDir, `${fileBase}.json`), body); +} + +beforeEach(() => { + graphsHome = mkdtempSync(join(tmpdir(), "graphs-home-")); + cwd = mkdtempSync(join(tmpdir(), "repo-")); + process.env.HIVEMIND_GRAPHS_HOME = graphsHome; +}); +afterEach(() => { + if (ORIG === undefined) delete process.env.HIVEMIND_GRAPHS_HOME; + else process.env.HIVEMIND_GRAPHS_HOME = ORIG; + rmSync(graphsHome, { recursive: true, force: true }); + rmSync(cwd, { recursive: true, force: true }); +}); + +describe("loadCurrentSnapshot", () => { + it("returns null when no graph has been built for the worktree", () => { + expect(loadCurrentSnapshot(cwd)).toBeNull(); + }); + + it("loads the snapshot the last build points at (keyed by commit_sha)", () => { + writeLastBuild(cwd, { ts: 1, commit_sha: "abc123", snapshot_sha256: "deadbeef" }); + writeSnapshotFile(cwd, "abc123", JSON.stringify({ nodes: [{ id: "x" }], links: [] })); + const snap = loadCurrentSnapshot(cwd); + expect(snap).not.toBeNull(); + expect(snap!.nodes).toHaveLength(1); + }); + + it("falls back to snapshot_sha256 when there is no commit context", () => { + writeLastBuild(cwd, { ts: 1, commit_sha: null, snapshot_sha256: "feedface" }); + writeSnapshotFile(cwd, "feedface", JSON.stringify({ nodes: [], links: [] })); + expect(loadCurrentSnapshot(cwd)).not.toBeNull(); + }); + + it("returns null when the snapshot file is missing on disk", () => { + writeLastBuild(cwd, { ts: 1, commit_sha: "abc123", snapshot_sha256: "d" }); + // no snapshot file written + expect(loadCurrentSnapshot(cwd)).toBeNull(); + }); + + it("returns null when the snapshot json is malformed or structurally wrong", () => { + writeLastBuild(cwd, { ts: 1, commit_sha: "abc123", snapshot_sha256: "d" }); + writeSnapshotFile(cwd, "abc123", "{ not valid json"); + expect(loadCurrentSnapshot(cwd)).toBeNull(); + + writeSnapshotFile(cwd, "abc123", JSON.stringify({ nope: true })); + expect(loadCurrentSnapshot(cwd)).toBeNull(); + }); +}); From eb92ff89e93bdd4512a12bd8881f3ececb060bb7 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:55:36 +0000 Subject: [PATCH 12/39] test: cover ensureDocsTable create and heal paths --- tests/shared/deeplake-api.test.ts | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/shared/deeplake-api.test.ts b/tests/shared/deeplake-api.test.ts index e1c14bbd..f3fcf70e 100644 --- a/tests/shared/deeplake-api.test.ts +++ b/tests/shared/deeplake-api.test.ts @@ -451,6 +451,7 @@ import { RULES_COLUMNS, GOALS_COLUMNS, KPIS_COLUMNS, + DOCS_COLUMNS, CODEBASE_COLUMNS, } from "../../src/deeplake-schema.js"; @@ -1043,6 +1044,65 @@ describe("DeeplakeApi.ensureGoalsTable", () => { }); }); +// ── ensureDocsTable ───────────────────────────────────────────────────────── + +describe("DeeplakeApi.ensureDocsTable", () => { + it("creates docs table when missing; heals after CREATE; emits (doc_id, version) index", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, status: 200, + json: async () => ({ tables: [] }), + }); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE TABLE + mockFetch.mockResolvedValueOnce(infoSchemaResponse(allOf(DOCS_COLUMNS))); // post-CREATE heal SELECT + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE INDEX (doc_id, version) + const api = makeApi(); + await api.ensureDocsTable("hivemind_docs"); + expect(mockFetch).toHaveBeenCalledTimes(4); + + const createSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(createSql).toContain(`CREATE TABLE IF NOT EXISTS "hivemind_docs"`); + expect(createSql).toContain("doc_id TEXT NOT NULL DEFAULT ''"); + expect(createSql).toContain("anchors TEXT NOT NULL DEFAULT '[]'"); + expect(createSql).toContain("tier TEXT NOT NULL DEFAULT 'fast'"); + expect(createSql).toContain("version BIGINT NOT NULL DEFAULT 1"); + expect(createSql).toContain("USING deeplake"); + + const indexSql = JSON.parse(mockFetch.mock.calls[3][1].body).query; + expect(indexSql).toContain(`"hivemind_docs"`); + expect(indexSql).toContain(`("doc_id", "version")`); + }); + + it("heals after CREATE: a missing column gets ALTERed before returning", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, status: 200, + json: async () => ({ tables: [] }), + }); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE (no-op vs legacy) + const legacy = allOf(DOCS_COLUMNS).filter(c => c !== "anchors"); + mockFetch.mockResolvedValueOnce(infoSchemaResponse(legacy)); // heal SELECT + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER anchors + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE INDEX + const api = makeApi(); + await api.ensureDocsTable("hivemind_docs"); + const alterSql = JSON.parse(mockFetch.mock.calls[3][1].body).query; + expect(alterSql).toBe(`ALTER TABLE "hivemind_docs" ADD COLUMN anchors TEXT NOT NULL DEFAULT '[]'`); + }); + + it("on existing up-to-date docs table: no CREATE TABLE / no ALTER (listTables + heal + index)", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, status: 200, + json: async () => ({ tables: [{ table_name: "hivemind_docs" }] }), + }); + mockFetch.mockResolvedValueOnce(infoSchemaResponse(allOf(DOCS_COLUMNS))); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE INDEX (doc_id, version) + const api = makeApi(); + await api.ensureDocsTable("hivemind_docs"); + const allSql = mockFetch.mock.calls.filter(c => c[1]?.body).map(c => JSON.parse(c[1].body).query).join(" | "); + expect(allSql).not.toContain("ALTER TABLE"); + expect(allSql).not.toContain("CREATE TABLE"); + }); +}); + // ── ensureKpisTable ───────────────────────────────────────────────────────── describe("DeeplakeApi.ensureKpisTable", () => { From 1de043e738a0517955ebd5f03501ff9e50afbf07 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 23 Jun 2026 16:57:46 +0000 Subject: [PATCH 13/39] fix: keep refresh dry-run read-only, unwrap model fences, boolean flag parsing --- src/commands/docs.ts | 14 ++++++++++---- src/docs/refresh-llm.ts | 15 ++++++++++++++- tests/shared/docs-refresh-llm.test.ts | 24 +++++++++++++++++++++++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index d4b6472c..81641912 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -123,6 +123,8 @@ function parseLimit(args: string[]): number { } const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor"]); +/** Flags that take NO value — they must not consume the following token. */ +const BOOLEAN_FLAGS = new Set(["--dry-run"]); /** Drop flag tokens (and their values) so positional scan sees only doc-id / content. */ function stripKnownFlags(args: string[]): string[] { @@ -130,7 +132,7 @@ function stripKnownFlags(args: string[]): string[] { for (let i = 0; i < args.length; i++) { const a = args[i]; if (KNOWN_FLAGS.has(a)) { - i++; + if (!BOOLEAN_FLAGS.has(a)) i++; // value-taking flags also skip their value continue; } if (KNOWN_FLAGS.has(a.split("=", 2)[0])) continue; @@ -172,9 +174,11 @@ export async function runDocsCommand(args: string[]): Promise { const query = api.query.bind(api); const pluginVersion = getVersion(); - // Only write subcommands need DDL — read-only show/list fall back to - // isMissingTableError so a fresh-install user doesn't pay a CREATE round-trip. - const WRITE_SUBS = new Set(["set", "archive", "refresh"]); + // Only write subcommands need DDL — read-only show/list (and refresh + // --dry-run) fall back to isMissingTableError so they don't pay a CREATE + // round-trip. `refresh` ensures the table itself, but only when it will + // actually write (handled inside its branch so --dry-run stays read-only). + const WRITE_SUBS = new Set(["set", "archive"]); if (WRITE_SUBS.has(sub)) { await api.ensureDocsTable(tableName); } @@ -332,6 +336,8 @@ export async function runDocsCommand(args: string[]): Promise { } return; } + // Real refresh writes — ensure the table now (dry-run above already returned). + await api.ensureDocsTable(tableName); const docsById = new Map(docs.map((d) => [d.doc_id, d])); const report = await refreshDocs({ query, diff --git a/src/docs/refresh-llm.ts b/src/docs/refresh-llm.ts index c5affffd..bf584d5c 100644 --- a/src/docs/refresh-llm.ts +++ b/src/docs/refresh-llm.ts @@ -17,6 +17,19 @@ import { buildClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; import { resolveCliBin } from "../utils/resolve-cli-bin.js"; import { buildRefreshPrompt, type GenerateFn } from "./refresh.js"; +/** + * Defensively unwrap the model's output. The prompt asks for raw markdown, + * but models sometimes wrap the whole body in a single ```fence``` (with an + * optional language tag). Strip exactly that outer fence so it doesn't leak + * into the stored doc; leave inner/partial fences untouched. Returns the + * trimmed body otherwise. + */ +export function unwrapModelOutput(raw: string): string { + const text = raw.trim(); + const fence = /^```[^\n]*\n([\s\S]*?)\n```$/.exec(text); + return fence ? fence[1].trim() : text; +} + /** Build a GenerateFn backed by the host `claude` CLI. */ export function makeClaudeGenerate(claudeBin?: string, timeoutMs = 120_000): GenerateFn { const bin = claudeBin ?? resolveCliBin("claude"); @@ -29,6 +42,6 @@ export function makeClaudeGenerate(claudeBin?: string, timeoutMs = 120_000): Gen timeout: timeoutMs, env: { ...process.env, HIVEMIND_WIKI_WORKER: "1", HIVEMIND_CAPTURE: "false" }, }); - return (out ?? "").toString().trim(); + return unwrapModelOutput((out ?? "").toString()); }; } diff --git a/tests/shared/docs-refresh-llm.test.ts b/tests/shared/docs-refresh-llm.test.ts index 27222444..7c78d464 100644 --- a/tests/shared/docs-refresh-llm.test.ts +++ b/tests/shared/docs-refresh-llm.test.ts @@ -6,7 +6,7 @@ vi.mock("node:child_process", () => ({ execFileSync: (...args: unknown[]) => execFileSyncMock(...args), })); -import { makeClaudeGenerate } from "../../src/docs/refresh-llm.js"; +import { makeClaudeGenerate, unwrapModelOutput } from "../../src/docs/refresh-llm.js"; import type { RefreshContext } from "../../src/docs/index.js"; const ctx: RefreshContext = { @@ -48,4 +48,26 @@ describe("makeClaudeGenerate", () => { const gen = makeClaudeGenerate("/usr/bin/claude"); expect(await gen(ctx)).toBe(""); }); + + it("unwraps an outer code fence the model may add around the whole body", async () => { + execFileSyncMock.mockReturnValue("```markdown\n# Doc\nbody\n```"); + const gen = makeClaudeGenerate("/usr/bin/claude"); + expect(await gen(ctx)).toBe("# Doc\nbody"); + }); +}); + +describe("unwrapModelOutput", () => { + it("strips a single outer fence with a language tag", () => { + expect(unwrapModelOutput("```md\nhello\nworld\n```")).toBe("hello\nworld"); + }); + it("strips a bare outer fence", () => { + expect(unwrapModelOutput("```\nhello\n```")).toBe("hello"); + }); + it("leaves un-fenced content (just trims)", () => { + expect(unwrapModelOutput(" # Title\nbody ")).toBe("# Title\nbody"); + }); + it("does NOT strip inner/partial fences in normal markdown", () => { + const md = "# Doc\n\n```ts\nconst x = 1;\n```\n\nmore"; + expect(unwrapModelOutput(md)).toBe(md); + }); }); From 8e5d220157b39f188402e2f90cc4bc3d39708fe1 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 24 Jun 2026 15:16:38 +0000 Subject: [PATCH 14/39] fix: short-circuit slow-tier and oversized edits before expensive work --- src/docs/gate.ts | 16 +++++++++++----- src/docs/refresh.ts | 12 ++++++++++++ tests/shared/docs-refresh.test.ts | 8 ++++++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/docs/gate.ts b/src/docs/gate.ts index 2cb6140d..06704d5d 100644 --- a/src/docs/gate.ts +++ b/src/docs/gate.ts @@ -77,12 +77,12 @@ export function countChangedLines(prev: string, next: string): number { /** Run the gate over a proposed doc edit. */ export function gateDocEdit(input: GateInput): GateResult { const reasons: string[] = []; - const changedLines = countChangedLines(input.prevContent, input.newContent); + const tooLong = input.newContent.length > GATE_MAX_CONTENT_LENGTH; if (input.newContent.length === 0) { reasons.push("proposed content is empty"); } - if (input.newContent.length > GATE_MAX_CONTENT_LENGTH) { + if (tooLong) { reasons.push(`proposed content exceeds ${GATE_MAX_CONTENT_LENGTH} chars (got ${input.newContent.length})`); } if (input.tier === "slow") { @@ -96,9 +96,15 @@ export function gateDocEdit(input: GateInput): GateResult { } } - const budget = input.maxChangedLines ?? DEFAULT_MAX_CHANGED_LINES; - if (changedLines > budget) { - reasons.push(`edit exceeds the bounded-change budget: ${changedLines} > ${budget} lines`); + // Skip the O(n*m) LCS diff when the content is already over the size cap — + // it would burn CPU on output that is guaranteed to be rejected anyway. + let changedLines = 0; + if (!tooLong) { + changedLines = countChangedLines(input.prevContent, input.newContent); + const budget = input.maxChangedLines ?? DEFAULT_MAX_CHANGED_LINES; + if (changedLines > budget) { + reasons.push(`edit exceeds the bounded-change budget: ${changedLines} > ${budget} lines`); + } } return { ok: reasons.length === 0, reasons, changedLines }; diff --git a/src/docs/refresh.ts b/src/docs/refresh.ts index cd0396c3..bd844144 100644 --- a/src/docs/refresh.ts +++ b/src/docs/refresh.ts @@ -141,6 +141,18 @@ export async function refreshDocs(args: RefreshArgs): Promise { continue; } + // Slow-tier docs are human-curated; the gate would reject any automatic + // edit anyway. Short-circuit BEFORE calling the LLM so we never spend a + // token on a guaranteed rejection or send protected content to the model. + if (doc.tier === "slow") { + outcomes.push({ + doc_id: imp.doc_id, + status: "rejected", + reasons: ["slow-tier docs are human-curated; automatic refresh is not allowed"], + }); + continue; + } + const newAnchors = reanchor(doc, nodeById, args.repoRoot); const changedSymbols = gatherChangedSymbols(imp.reasons, nodeById, args.repoRoot); diff --git a/tests/shared/docs-refresh.test.ts b/tests/shared/docs-refresh.test.ts index 2db48159..a7e8b651 100644 --- a/tests/shared/docs-refresh.test.ts +++ b/tests/shared/docs-refresh.test.ts @@ -181,14 +181,18 @@ describe("refreshDocs", () => { expect(calls).toHaveLength(0); // nothing written }); - it("rejects a slow-tier doc outright", async () => { + it("rejects a slow-tier doc WITHOUT calling the LLM (no token spend, no leak)", async () => { const d = doc({ tier: "slow", anchors: [{ symbol_id: foo.id, content_hash: "x" }] }); const { calls, query } = mockQuery([]); + const generate = vi.fn(async () => "small"); const report = await refreshDocs({ query, tableName: "hivemind_docs", snap: s, repoRoot: dir, - impacted: impacted(), docsById: new Map([["a.ts", d]]), generate: async () => "small", + impacted: impacted(), docsById: new Map([["a.ts", d]]), generate, }); expect(report.rejected).toBe(1); + expect(report.outcomes[0].reasons?.join()).toMatch(/slow-tier/); + // The generator is never invoked for slow-tier docs. + expect(generate).not.toHaveBeenCalled(); expect(calls).toHaveLength(0); }); From 6783553ed21bb3370bca440024cdb87aecb60931 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 24 Jun 2026 15:16:43 +0000 Subject: [PATCH 15/39] fix: propagate project on doc version bumps --- src/docs/write.ts | 6 +++++- tests/shared/docs.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/docs/write.ts b/src/docs/write.ts index 04d6c056..0c1cad40 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -74,6 +74,8 @@ export interface EditDocInput { status?: "active" | "archived"; /** New VFS path. Omit to keep the previous path. */ path?: string; + /** New project. Omit to keep the previous project. */ + project?: string; agent?: string; plugin_version?: string; } @@ -198,6 +200,7 @@ export async function setDoc( tier: input.tier, status: input.status, path: input.path, + project: input.project, agent: input.agent, plugin_version: input.plugin_version, }); @@ -238,6 +241,7 @@ async function appendVersion( const tier: DocTier = next.tier ?? previous.tier; const status = next.status ?? (previous.status as "active" | "archived"); const path = next.path ?? previous.path; + const project = next.project ?? previous.project; const sql = `INSERT INTO "${safe}" ` + @@ -251,7 +255,7 @@ async function appendVersion( `E'${sqlStr(anchors)}', ` + `'${sqlStr(tier)}', ` + `'${sqlStr(status)}', ` + - `'${sqlStr(previous.project)}', ` + + `'${sqlStr(project)}', ` + `${nextVersion}, ` + // created_at carried from the original row — immutable creation stamp. `'${sqlStr(previous.created_at)}', ` + diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 7c7c4695..3bf99085 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -250,6 +250,21 @@ describe("setDoc", () => { expect(calls[1]).toMatch(/, 1, /); }); + it("propagates a new project on a version bump (not frozen at the old value)", async () => { + const { calls, query } = mockQuery([ + () => [fakeRow({ doc_id: "src/a.ts", version: 1, project: "old-proj" })], + () => [], + ]); + await setDoc(query, TBL, { + doc_id: "src/a.ts", + path: "/docs/p/a.ts.md", + content: "updated", + project: "new-proj", + }); + expect(calls[1]).toContain("'new-proj'"); + expect(calls[1]).not.toContain("'old-proj'"); + }); + it("APPENDS version+1 when the doc_id already exists — never forks a second v1", async () => { const { calls, query } = mockQuery([ () => [fakeRow({ doc_id: "src/a.ts", version: 4, created_at: "2026-01-01T00:00:00.000Z" })], From 4b7ed157edac5e842da7f925ce80e8343c96b360 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 24 Jun 2026 15:16:48 +0000 Subject: [PATCH 16/39] fix: guard docs set content read inside error handler --- src/commands/docs.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 81641912..cf5ae510 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -192,7 +192,6 @@ export async function runDocsCommand(args: string[]): Promise { throw new Error("unreachable"); } const project = flagValue(args, "--project") ?? ""; - const content = resolveContent(positional[1], args); const path = flagValue(args, "--path") ?? defaultVfsPath(project, docId); const tier = parseTier(args); // Optional anchors: build from the current graph so the doc is tied to the @@ -225,6 +224,9 @@ export async function runDocsCommand(args: string[]): Promise { } } try { + // resolveContent reads --file / stdin and can throw on a bad path — keep + // it inside the guard so it surfaces as a controlled CLI error. + const content = resolveContent(positional[1], args); const out = await setDoc(query, tableName, { doc_id: docId, path, From c7ec190e975753d390b0a4e8f3fe0202dc61ac53 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 30 Jun 2026 14:38:31 +0000 Subject: [PATCH 17/39] fix: add read-stability gate for deeplake partial reads --- src/docs/read.ts | 44 ++++++++----- src/docs/stable-read.ts | 91 +++++++++++++++++++++++++++ tests/shared/docs-stable-read.test.ts | 77 +++++++++++++++++++++++ 3 files changed, 198 insertions(+), 14 deletions(-) create mode 100644 src/docs/stable-read.ts create mode 100644 tests/shared/docs-stable-read.test.ts diff --git a/src/docs/read.ts b/src/docs/read.ts index 7313a32e..a5e14d70 100644 --- a/src/docs/read.ts +++ b/src/docs/read.ts @@ -14,6 +14,7 @@ */ import { sqlIdent, sqlStr } from "../utils/sql.js"; +import { stableUnionRows } from "./stable-read.js"; export type QueryFn = (sql: string) => Promise>>; @@ -72,11 +73,12 @@ export async function listDocs( opts: ListDocsOpts = {}, ): Promise { const safe = sqlIdent(tableName); - // ORDER BY version DESC primes the JS dedup: the first row seen per - // doc_id is the winning latest-version row. updated_at DESC then id DESC - // are deterministic tie-breakers so listDocs and getDocLatest agree on - // the winner row-for-row under same-millisecond / race duplicates. - const rows = await query( + // Read through the stability gate: the Deeplake backend can return a partial + // row set right after writes, which would make a refresh silently skip stale + // docs. stableUnionRows re-reads until the union converges so we see EVERY + // row. (ORDER BY is moot through the union — we re-sort after dedup below.) + const rows = await stableUnionRows( + query, `SELECT ${SELECT_COLS} FROM "${safe}" ORDER BY version DESC, updated_at DESC, id DESC`, ); @@ -111,16 +113,30 @@ export async function getDocLatest( docId: string, ): Promise { const safe = sqlIdent(tableName); - // version DESC picks the highest version; updated_at DESC then id DESC - // break ties deterministically when concurrent edits produced duplicate - // v=N+1 rows for the same doc_id — keeps this in agreement with listDocs. - const rows = await query( - `SELECT ${SELECT_COLS} FROM "${safe}" ` + - `WHERE doc_id = '${sqlStr(docId)}' ` + - `ORDER BY version DESC, updated_at DESC, id DESC LIMIT 1`, + // Read ALL version rows for this doc through the stability gate, then pick + // the latest in JS. A bare `... ORDER BY version DESC LIMIT 1` is unsafe on + // this backend: a partial read can return an OLD version as "latest" (or + // zero rows) right after a write. Unioning every version row and choosing + // the max guarantees we never resolve to a stale version or miss the doc. + const raw = await stableUnionRows( + query, + `SELECT ${SELECT_COLS} FROM "${safe}" WHERE doc_id = '${sqlStr(docId)}'`, ); - if (rows.length === 0) return null; - return normalize(rows[0]); + let best: DocRow | null = null; + for (const r of raw) { + const row = normalize(r); + if (!row) continue; + if ( + best === null || + row.version > best.version || + (row.version === best.version && + (row.updated_at.localeCompare(best.updated_at) > 0 || + (row.updated_at === best.updated_at && row.id.localeCompare(best.id) > 0))) + ) { + best = row; + } + } + return best; } /** Parse the `anchors` JSON cell into a typed array; degrade to [] on garbage. */ diff --git a/src/docs/stable-read.ts b/src/docs/stable-read.ts new file mode 100644 index 00000000..533ae4f4 --- /dev/null +++ b/src/docs/stable-read.ts @@ -0,0 +1,91 @@ +/** + * Read-stability gate for the Deeplake SQL backend. + * + * The backend exhibits read-after-write inconsistency: for a window after a + * burst of INSERTs, the SAME `SELECT` returns a NON-DETERMINISTIC PARTIAL + * subset of the rows (the backend itself reports the partial count). It + * eventually converges, but any read in that window can silently miss rows — + * which for the doc system means `refresh` skips stale docs, and a latest-row + * read can resolve to an OLD version. (Repro: deeplake-readafterwrite-repro.mjs.) + * + * Key property we exploit: every partial read is a SUBSET of the true row set + * (counts observed range from partial up to the true count, never above). So + * the UNION of rows across repeated reads monotonically approaches the + * complete set. We re-read until the union stops growing for `stableReads` + * consecutive reads (converged), or `maxReads` is hit, and return the union. + * + * "Two consecutive reads agree" is NOT used — two consecutive identical + * partial reads happen often (e.g. 4,4) and would lock onto a partial set. + * Union-until-stable cannot under-return for rows that appeared in any read. + */ + +export type QueryFn = (sql: string) => Promise>>; + +export interface StableReadOpts { + /** Unique row-identity column to union on. Default "id". */ + idKey?: string; + /** + * Consecutive non-growing reads required to call it converged. Default 3. + * Higher is safer: a partial result can REPEAT (e.g. 5,5,5 with a true + * count of 8), so a very low value risks locking onto a partial before a + * fuller read arrives. The union only grows, so a few confirmations suffice. + */ + stableReads?: number; + /** Hard cap on reads. Default 10. */ + maxReads?: number; + /** Extra delay between reads (ms). Default 0 — HTTP round-trips already space them. */ + delayMs?: number; + /** Optional logger. */ + log?: (msg: string) => void; + /** Injectable sleep for tests. */ + sleep?: (ms: number) => Promise; +} + +const defaultSleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +/** + * Run `sql` repeatedly and return the UNION of all distinct rows seen (keyed + * by `idKey`). Stops early once the union has not grown for `stableReads` + * consecutive reads. Returns as soon as the first read already looks stable + * on a tiny set, but always does at least `stableReads` reads so a partial + * first read can't end the loop prematurely. + */ +export async function stableUnionRows( + query: QueryFn, + sql: string, + opts: StableReadOpts = {}, +): Promise>> { + const idKey = opts.idKey ?? "id"; + const stableReads = Math.max(1, opts.stableReads ?? 3); + const maxReads = Math.max(stableReads, opts.maxReads ?? 10); + // Each read is a separate HTTP round-trip (~hundreds of ms), which already + // spaces the reads out — an extra artificial delay just adds latency for no + // consistency benefit. Default 0; callers can set one if a backend ever + // needs explicit backoff. + const delayMs = opts.delayMs ?? 0; + const sleep = opts.sleep ?? defaultSleep; + + const union = new Map>(); + let stableStreak = 0; + let reads = 0; + + while (reads < maxReads) { + const rows = await query(sql); + reads++; + let grew = false; + for (const row of rows) { + const k = String(row[idKey] ?? ""); + if (k === "") continue; // can't union rows without an identity key + if (!union.has(k)) { + union.set(k, row); + grew = true; + } + } + stableStreak = grew ? 0 : stableStreak + 1; + if (stableStreak >= stableReads) break; + if (reads < maxReads) await sleep(delayMs); + } + + opts.log?.(`stable-read: ${union.size} rows after ${reads} reads (streak ${stableStreak})`); + return [...union.values()]; +} diff --git a/tests/shared/docs-stable-read.test.ts b/tests/shared/docs-stable-read.test.ts new file mode 100644 index 00000000..46df0845 --- /dev/null +++ b/tests/shared/docs-stable-read.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from "vitest"; +import { stableUnionRows } from "../../src/docs/stable-read.js"; + +const noSleep = async () => {}; + +/** A query that returns a programmed sequence of row-sets, one per call. */ +function scriptedQuery(sequence: Array>>) { + let i = 0; + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + const out = sequence[Math.min(i, sequence.length - 1)]; + i++; + return out; + }); + return { query, calls }; +} + +const rows = (...ids: number[]) => ids.map((n) => ({ id: `r${n}`, n })); + +describe("stableUnionRows", () => { + it("unions partial reads into the COMPLETE set (defeats the partial-read bug)", async () => { + // Simulated backend: returns partial subsets, only occasionally the full 8. + const { query } = scriptedQuery([ + rows(0, 1, 2, 3, 4), // 5 + rows(0, 1, 2, 3, 4), // 5 (repeat — must NOT stop here) + rows(0, 1, 2, 3, 4, 5, 6, 7), // 8 (full) + rows(0, 1, 2, 3), // 4 (regression — union must not shrink) + rows(0, 1, 2, 3, 4, 5, 6, 7), + rows(0, 1, 2, 3, 4, 5, 6, 7), + rows(0, 1, 2, 3, 4, 5, 6, 7), + ]); + const out = await stableUnionRows(query, "SELECT ...", { sleep: noSleep, stableReads: 3, maxReads: 12 }); + expect(out).toHaveLength(8); + expect(out.map((r) => r.id).sort()).toEqual(["r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7"]); + }); + + it("does NOT lock onto a repeated partial result", async () => { + // 5 appears 3x before the full 8 — a 'two/three consecutive agree' strategy + // would wrongly return 5. Union-until-stable must still reach 8. + const { query } = scriptedQuery([ + rows(0, 1, 2, 3, 4), + rows(0, 1, 2, 3, 4), + rows(0, 1, 2, 3, 4), + rows(0, 1, 2, 3, 4, 5, 6, 7), + rows(0, 1, 2, 3, 4, 5, 6, 7), + rows(0, 1, 2, 3, 4, 5, 6, 7), + rows(0, 1, 2, 3, 4, 5, 6, 7), + ]); + const out = await stableUnionRows(query, "SELECT ...", { sleep: noSleep, stableReads: 3, maxReads: 12 }); + expect(out).toHaveLength(8); + }); + + it("converges quickly when reads are already complete and stable", async () => { + const { query, calls } = scriptedQuery([rows(0, 1, 2)]); + const out = await stableUnionRows(query, "SELECT ...", { sleep: noSleep, stableReads: 3, maxReads: 12 }); + expect(out).toHaveLength(3); + // first read grows, then needs `stableReads` non-growing reads -> 4 total. + expect(calls.length).toBe(4); + }); + + it("respects the maxReads cap and returns the best union so far", async () => { + // Never returns the full set within the cap; still returns what it unioned. + const { query, calls } = scriptedQuery([ + rows(0), rows(1), rows(2), rows(3), rows(4), + ]); + const out = await stableUnionRows(query, "SELECT ...", { sleep: noSleep, stableReads: 3, maxReads: 5 }); + expect(calls.length).toBe(5); + expect(out.length).toBe(5); // unioned r0..r4 across the 5 capped reads + }); + + it("skips rows missing the identity key", async () => { + const { query } = scriptedQuery([[{ id: "r0" }, { nope: 1 }, { id: "" }]]); + const out = await stableUnionRows(query, "SELECT ...", { sleep: noSleep, stableReads: 2, maxReads: 4 }); + expect(out.map((r) => r.id)).toEqual(["r0"]); + }); +}); From 16718c79f7ca40f2163b449a7252197b9cd10600 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 30 Jun 2026 14:38:36 +0000 Subject: [PATCH 18/39] test: stub read-stability gate in doc unit tests --- tests/claude-code/cli-docs.test.ts | 5 +++++ tests/shared/docs-refresh.test.ts | 5 +++++ tests/shared/docs.test.ts | 22 +++++++++++++++++----- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/claude-code/cli-docs.test.ts b/tests/claude-code/cli-docs.test.ts index 94b1b03f..801a75c8 100644 --- a/tests/claude-code/cli-docs.test.ts +++ b/tests/claude-code/cli-docs.test.ts @@ -11,6 +11,11 @@ import { tmpdir } from "node:os"; * LLM). The store functions run for real against the query spy. */ +// Stub the read-stability gate to a single pass-through query (see docs.test.ts). +vi.mock("../../src/docs/stable-read.js", () => ({ + stableUnionRows: (q: (sql: string) => unknown, sql: string) => q(sql), +})); + const ensureDocsTableMock = vi.fn(); const queryMock = vi.fn(); const loadCurrentSnapshotMock = vi.fn(); diff --git a/tests/shared/docs-refresh.test.ts b/tests/shared/docs-refresh.test.ts index a7e8b651..db4cd7bb 100644 --- a/tests/shared/docs-refresh.test.ts +++ b/tests/shared/docs-refresh.test.ts @@ -1,4 +1,9 @@ import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; + +// Stub the read-stability gate to a single pass-through query (see docs.test.ts). +vi.mock("../../src/docs/stable-read.js", () => ({ + stableUnionRows: (q: (sql: string) => unknown, sql: string) => q(sql), +})); import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 3bf99085..3d7078b9 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -1,4 +1,12 @@ import { describe, expect, it, beforeEach, vi } from "vitest"; + +// The read-stability gate (stableUnionRows) re-reads with delays in production +// to defeat the Deeplake partial-read bug. In these unit tests we stub it to a +// single pass-through query so call-count / SQL-shape assertions stay exact and +// fast. The gate's own behavior is covered in docs-stable-read.test.ts. +vi.mock("../../src/docs/stable-read.js", () => ({ + stableUnionRows: (q: (sql: string) => unknown, sql: string) => q(sql), +})); import { insertDoc, editDoc, @@ -170,7 +178,7 @@ describe("editDoc", () => { const result = await editDoc(query, TBL, { doc_id: "src/shell/deeplake-fs.ts", content: "new" }); expect(result).toEqual({ doc_id: "src/shell/deeplake-fs.ts", version: 2 }); expect(calls).toHaveLength(2); - expect(calls[0]).toMatch(/^SELECT .* FROM "hivemind_docs" WHERE doc_id = 'src\/shell\/deeplake-fs.ts' ORDER BY version DESC, updated_at DESC, id DESC LIMIT 1$/); + expect(calls[0]).toMatch(/^SELECT .* FROM "hivemind_docs" WHERE doc_id = 'src\/shell\/deeplake-fs.ts'$/); expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); expect(calls[1]).toContain(`E'new'`); expect(calls[1]).toContain(", 2, "); @@ -390,16 +398,20 @@ describe("listDocs", () => { // ── getDocLatest ────────────────────────────────────────────────────────────── describe("getDocLatest", () => { - it("returns the single latest row, LIMIT 1, escaped doc_id", async () => { + it("returns the latest row, picking max version in JS, escaped doc_id", async () => { + // Reads ALL version rows for the doc (no LIMIT 1 — unsafe on this backend) + // and picks the max version in JS. const { calls, query } = mockQuery([ - () => [fakeRow({ doc_id: "X", version: 5, content: "current" })], + () => [ + fakeRow({ id: "r1", doc_id: "X", version: 3, content: "old" }), + fakeRow({ id: "r2", doc_id: "X", version: 5, content: "current" }), + ], ]); const row = await getDocLatest(query, TBL, "X"); expect(row?.version).toBe(5); expect(row?.content).toBe("current"); - expect(calls[0]).toMatch(/LIMIT 1$/); expect(calls[0]).toContain(`doc_id = 'X'`); - expect(calls[0]).toContain("ORDER BY version DESC, updated_at DESC, id DESC"); + expect(calls[0]).not.toMatch(/LIMIT 1/); }); it("returns null when nothing matches", async () => { From 2d17808b8d4eb6d52e963ba0db93ac86e2b88209 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 30 Jun 2026 16:57:19 +0000 Subject: [PATCH 19/39] fix: normalize source before hashing to reduce doc churn --- src/docs/anchors.ts | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/docs/anchors.ts b/src/docs/anchors.ts index 330d9599..bc2b8adb 100644 --- a/src/docs/anchors.ts +++ b/src/docs/anchors.ts @@ -51,15 +51,42 @@ export function readSymbolSource(node: GraphNode, repoRoot: string): string | nu return lines.slice(loc.startLine - 1, loc.endLine).join("\n"); } -/** sha256 of a source slice. Line endings are already normalized by the splitter. */ -export function hashSource(src: string): string { - return createHash("sha256").update(src).digest("hex"); +/** + * Normalize a source slice before hashing so cosmetic edits don't churn docs. + * + * Strips comments, trailing whitespace, and blank lines — these change the + * bytes but not what the doc describes, so a reformat / comment edit / blank-line + * shuffle should NOT mark a doc stale. Indentation is PRESERVED (significant in + * Python). This is the main lever against false-positive rewrites. + * + * It does not normalize identifiers — renaming a symbol still changes the hash + * (a genuine code edit); whether that should flag the doc is left to the gate / + * human, not hidden here. + */ +export function normalizeForHash(src: string, language?: string): string { + let s = src; + if (language === "python" || language === "ruby") { + s = s.replace(/#.*$/gm, ""); // line comments + } else { + s = s.replace(/\/\*[\s\S]*?\*\//g, ""); // block comments + s = s.replace(/\/\/.*$/gm, ""); // line comments + } + return s + .split(/\r?\n/) + .map((l) => l.replace(/\s+$/, "")) // trailing whitespace + .filter((l) => l.trim() !== "") // blank lines + .join("\n"); +} + +/** sha256 of a source slice, after comment/whitespace normalization. */ +export function hashSource(src: string, language?: string): string { + return createHash("sha256").update(normalizeForHash(src, language)).digest("hex"); } /** Compute the content hash for a symbol, or null if its source can't be read. */ export function computeSymbolHash(node: GraphNode, repoRoot: string): string | null { const src = readSymbolSource(node, repoRoot); - return src === null ? null : hashSource(src); + return src === null ? null : hashSource(src, node.language); } /** Build an anchor for a symbol node, or null if its source can't be read. */ From 9c2d8781a09369a270b09eb3788759d1988be11a Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Tue, 30 Jun 2026 16:57:24 +0000 Subject: [PATCH 20/39] feat: add selective docs generate command --- src/commands/docs.ts | 69 +++++++- src/docs/generate.ts | 244 +++++++++++++++++++++++++++++ src/docs/refresh-llm.ts | 40 +++-- tests/shared/docs-generate.test.ts | 152 ++++++++++++++++++ 4 files changed, 489 insertions(+), 16 deletions(-) create mode 100644 src/docs/generate.ts create mode 100644 tests/shared/docs-generate.test.ts diff --git a/src/commands/docs.ts b/src/commands/docs.ts index cf5ae510..d8902504 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -39,7 +39,8 @@ import { type DocTier, type DocAnchor, } from "../docs/index.js"; -import { makeClaudeGenerate } from "../docs/refresh-llm.js"; +import { makeClaudeGenerate, makeClaudeGenerateDoc } from "../docs/refresh-llm.js"; +import { generateDocs, selectTargets, type GenScope } from "../docs/generate.js"; import { loadCurrentSnapshot } from "../graph/load-current.js"; import { isMissingTableError } from "../deeplake-schema.js"; @@ -55,6 +56,13 @@ Usage: Detect docs whose anchored code drifted (vs the current graph) and regenerate them via the host LLM, gated. --dry-run only reports the impacted docs without calling the LLM or writing anything. + hivemind docs generate [--cwd ] [--scope file|symbol] [--include ] + [--exclude ] [--limit N] [--concurrency N] + [--force] [--dry-run] + Auto-author docs for the codebase from the AST graph (which already skips + .gitignored / non-code files). Default scope=file (one doc per file, + anchored to its symbols). Skips files that already have a doc unless + --force. --dry-run lists the targets without calling the LLM. `.trim(); function requireConfig(): NonNullable> { @@ -122,9 +130,9 @@ function parseLimit(args: string[]): number { return n; } -const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor"]); +const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor", "--scope", "--include", "--exclude", "--concurrency", "--force"]); /** Flags that take NO value — they must not consume the following token. */ -const BOOLEAN_FLAGS = new Set(["--dry-run"]); +const BOOLEAN_FLAGS = new Set(["--dry-run", "--force"]); /** Drop flag tokens (and their values) so positional scan sees only doc-id / content. */ function stripKnownFlags(args: string[]): string[] { @@ -360,6 +368,61 @@ export async function runDocsCommand(args: string[]): Promise { return; } + if (sub === "generate") { + const cwd = flagValue(args, "--cwd") ?? process.cwd(); + const dryRun = args.includes("--dry-run"); + const force = args.includes("--force"); + const scopeRaw = flagValue(args, "--scope") ?? "file"; + if (scopeRaw !== "file" && scopeRaw !== "symbol") { + console.error("Invalid --scope. Allowed: file | symbol."); + process.exit(1); + throw new Error("unreachable"); + } + const scope = scopeRaw as GenScope; + const include = flagValues(args, "--include"); + const exclude = flagValues(args, "--exclude"); + const limitRaw = flagValue(args, "--limit"); + const limit = limitRaw === undefined ? undefined : Number(limitRaw); + const concurrency = Number(flagValue(args, "--concurrency") ?? "6"); + const project = flagValue(args, "--project") ?? ""; + + const snap = loadCurrentSnapshot(cwd); + if (!snap) { + console.error("No local graph for this directory. Run `hivemind graph build` first."); + process.exit(1); + throw new Error("unreachable"); + } + let existingDocs: DocRow[] = []; + try { + existingDocs = await listDocs(query, tableName, { status: "all", limit: 1000000 }); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + const existing = new Set(existingDocs.map((d) => d.doc_id)); + const allTargets = selectTargets(snap, { scope, include, exclude }); + const todo = force ? allTargets : allTargets.filter((t) => !existing.has(t.doc_id)); + + if (dryRun) { + console.log(`${todo.length} target(s) would be documented (scope=${scope}); ${allTargets.length - todo.length} already documented or skipped.`); + for (const t of todo.slice(0, limit ?? 60)) { + console.log(` ${t.doc_id} (${t.symbols.length} symbols)`); + } + return; + } + + await api.ensureDocsTable(tableName); + const report = await generateDocs({ + query, tableName, snap, repoRoot: cwd, project, scope, include, exclude, + existing, force, limit, concurrency, + generate: makeClaudeGenerateDoc(), agent: cfg.userName, pluginVersion, + }); + console.log(`Generated ${report.created}, skipped ${report.skipped}, failed ${report.failed} (of ${report.targets} targets).`); + for (const o of report.outcomes) { + if (o.status !== "created") console.log(` ${o.status} ${o.doc_id}: ${o.reason ?? ""}`); + } + return; + } + console.error(`Unknown subcommand: ${sub}`); console.error(USAGE); process.exit(1); diff --git a/src/docs/generate.ts b/src/docs/generate.ts new file mode 100644 index 00000000..29380238 --- /dev/null +++ b/src/docs/generate.ts @@ -0,0 +1,244 @@ +/** + * Doc generation — author docs for a real codebase, selectively. + * + * Walks the AST graph (which already excludes .gitignored files, node_modules, + * env files, etc. — see graph/ignore-config.ts), so non-code junk is filtered + * for free. On top of that we apply doc-specific include/exclude globs and a + * sensible default exclude list (tests, type decls, config, barrels) so we + * don't document files nobody needs. + * + * Two scopes: + * - `file` (default): ONE doc per source file, anchored to all of its + * documentable symbols (a small "component" doc — fewer docs, higher value). + * - `symbol`: one doc per function/class/method. + * + * Idempotent: a target that already has a doc is skipped unless `force`. The + * orchestrator is pure except for the injected `generate` (LLM) and `query`. + */ + +import { buildAnchor, readSymbolSource } from "./anchors.js"; +import { setDoc } from "./write.js"; +import type { DocAnchor, QueryFn } from "./read.js"; +import type { GraphNode, GraphSnapshot } from "../graph/types.js"; + +/** Kinds worth a doc. const/module/variable are skipped as low-signal noise. */ +const DOCUMENTABLE_KINDS = new Set(["function", "class", "method", "interface", "type_alias", "enum"]); + +/** Doc-specific default excludes (on top of the graph's gitignore filtering). */ +export const DEFAULT_EXCLUDE_GLOBS = [ + "**/*.test.*", + "**/*.spec.*", + "**/*.d.ts", + "**/*.config.*", + "**/index.ts", + "**/index.js", +]; + +export type GenScope = "file" | "symbol"; + +export interface GenTarget { + /** Stable doc_id: the source file path (file scope) or the symbol id (symbol scope). */ + doc_id: string; + /** Source file path. */ + file: string; + /** Documentable symbols to anchor + describe. */ + symbols: GraphNode[]; +} + +export interface GenDocInput { + doc_id: string; + file: string; + symbols: Array<{ id: string; signature?: string; source: string }>; +} +export type GenerateDocFn = (input: GenDocInput) => Promise; + +/** Convert a glob (`*`, `**`, `?`) to an anchored RegExp over forward-slash paths. */ +export function globToRegExp(glob: string): RegExp { + let re = ""; + for (let i = 0; i < glob.length; i++) { + const c = glob[i]; + if (c === "*") { + if (glob[i + 1] === "*") { + re += ".*"; // ** → any (including /) + i++; + if (glob[i + 1] === "/") i++; // swallow the slash after ** + } else { + re += "[^/]*"; // * → any except / + } + } else if (c === "?") { + re += "[^/]"; + } else if (".+^${}()|[]\\".includes(c)) { + re += "\\" + c; + } else { + re += c; + } + } + return new RegExp(`^${re}$`); +} + +function matchesAny(path: string, globs: string[]): boolean { + return globs.some((g) => globToRegExp(g).test(path)); +} + +/** + * Pick the targets to document from the snapshot, applying scope + include/exclude. + * `exclude` is ADDITIVE to DEFAULT_EXCLUDE_GLOBS; `include` (if given) keeps only + * matching files. + */ +export function selectTargets( + snap: GraphSnapshot, + opts: { scope?: GenScope; include?: string[]; exclude?: string[] } = {}, +): GenTarget[] { + const scope = opts.scope ?? "file"; + const excludes = [...DEFAULT_EXCLUDE_GLOBS, ...(opts.exclude ?? [])]; + const includes = opts.include ?? []; + + const docNodes = snap.nodes.filter((n) => DOCUMENTABLE_KINDS.has(n.kind)); + const keep = (file: string): boolean => { + if (includes.length > 0 && !matchesAny(file, includes)) return false; + if (matchesAny(file, excludes)) return false; + return true; + }; + + if (scope === "symbol") { + return docNodes + .filter((n) => keep(n.source_file)) + .map((n) => ({ doc_id: n.id, file: n.source_file, symbols: [n] })); + } + + // file scope: group documentable symbols by source_file. + const byFile = new Map(); + for (const n of docNodes) { + if (!keep(n.source_file)) continue; + const list = byFile.get(n.source_file); + if (list) list.push(n); + else byFile.set(n.source_file, [n]); + } + return [...byFile.entries()] + .map(([file, symbols]) => ({ doc_id: file, file, symbols })) + .sort((a, b) => a.doc_id.localeCompare(b.doc_id)); +} + +/** Prompt for a fresh doc — concise, code-grounded, markdown-only. */ +export function buildGeneratePrompt(input: GenDocInput): string { + const syms = input.symbols + .map((s) => `### ${s.id}\n${s.signature ? s.signature + "\n" : ""}\n\`\`\`\n${s.source}\n\`\`\``) + .join("\n\n"); + return [ + "You are writing concise internal documentation for ONE source file.", + "Describe what the file is for and what its key symbols do — one short line per symbol.", + "Be precise and grounded in the code below. Output ONLY markdown, no preamble, no outer code fence. Keep it under ~1500 characters.", + "", + `## File: ${input.file}`, + "", + "## Symbols", + syms || "(none)", + ].join("\n"); +} + +export interface GenerateArgs { + query: QueryFn; + tableName: string; + snap: GraphSnapshot; + repoRoot: string; + project?: string; + scope?: GenScope; + include?: string[]; + exclude?: string[]; + /** doc_ids that already exist (skipped unless force). */ + existing: Set; + force?: boolean; + limit?: number; + concurrency?: number; + generate: GenerateDocFn; + agent?: string; + pluginVersion?: string; +} + +export interface GenOutcome { + doc_id: string; + status: "created" | "skipped" | "failed"; + reason?: string; +} +export interface GenReport { + outcomes: GenOutcome[]; + targets: number; + created: number; + skipped: number; + failed: number; +} + +/** Run `fn` over `items` with at most `n` in flight. */ +async function runPool(items: T[], n: number, fn: (item: T) => Promise): Promise { + let i = 0; + const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => { + while (i < items.length) { + const idx = i++; + await fn(items[idx]); + } + }); + await Promise.all(workers); +} + +function defaultVfsPath(project: string, docId: string): string { + return `/docs/${project || "default"}/${docId}.md`; +} + +/** Generate docs for the selected targets. */ +export async function generateDocs(args: GenerateArgs): Promise { + const project = args.project ?? ""; + let targets = selectTargets(args.snap, { scope: args.scope, include: args.include, exclude: args.exclude }); + if (!args.force) targets = targets.filter((t) => !args.existing.has(t.doc_id)); + if (args.limit !== undefined) targets = targets.slice(0, args.limit); + + const outcomes: GenOutcome[] = []; + await runPool(targets, args.concurrency ?? 6, async (t) => { + // Build the prompt context from current source; also pre-build anchors. + const symInput: GenDocInput["symbols"] = []; + const anchors: DocAnchor[] = []; + for (const n of t.symbols) { + const source = readSymbolSource(n, args.repoRoot); + const anchor = buildAnchor(n, args.repoRoot); + if (source !== null) symInput.push({ id: n.id, signature: n.signature, source }); + if (anchor) anchors.push(anchor); + } + if (anchors.length === 0) { + outcomes.push({ doc_id: t.doc_id, status: "skipped", reason: "no readable symbols to anchor" }); + return; + } + let content: string; + try { + content = await args.generate({ doc_id: t.doc_id, file: t.file, symbols: symInput }); + } catch (err) { + outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `generate failed: ${(err as Error).message}` }); + return; + } + if (content.trim() === "") { + outcomes.push({ doc_id: t.doc_id, status: "failed", reason: "empty content" }); + return; + } + try { + await setDoc(args.query, args.tableName, { + doc_id: t.doc_id, + path: defaultVfsPath(project, t.doc_id), + content, + anchors, + tier: "fast", + project, + agent: args.agent ?? "docs-generate", + plugin_version: args.pluginVersion, + }); + outcomes.push({ doc_id: t.doc_id, status: "created" }); + } catch (err) { + outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `write failed: ${(err as Error).message}` }); + } + }); + + return { + outcomes, + targets: targets.length, + created: outcomes.filter((o) => o.status === "created").length, + skipped: outcomes.filter((o) => o.status === "skipped").length, + failed: outcomes.filter((o) => o.status === "failed").length, + }; +} diff --git a/src/docs/refresh-llm.ts b/src/docs/refresh-llm.ts index bf584d5c..b0a4fb81 100644 --- a/src/docs/refresh-llm.ts +++ b/src/docs/refresh-llm.ts @@ -16,6 +16,7 @@ import { execFileSync } from "node:child_process"; import { buildClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; import { resolveCliBin } from "../utils/resolve-cli-bin.js"; import { buildRefreshPrompt, type GenerateFn } from "./refresh.js"; +import { buildGeneratePrompt, type GenerateDocFn } from "./generate.js"; /** * Defensively unwrap the model's output. The prompt asks for raw markdown, @@ -30,18 +31,31 @@ export function unwrapModelOutput(raw: string): string { return fence ? fence[1].trim() : text; } -/** Build a GenerateFn backed by the host `claude` CLI. */ +/** Run a single prompt through the host `claude` CLI and return the unwrapped output. */ +export function runClaudePrompt(bin: string, prompt: string, timeoutMs = 120_000): string { + const inv = buildClaudeInvocation(bin, prompt); + const out = execFileSync(inv.file, inv.args, { + ...inv.options, + encoding: "utf-8", + timeout: timeoutMs, + env: { ...process.env, HIVEMIND_WIKI_WORKER: "1", HIVEMIND_CAPTURE: "false" }, + }); + return unwrapModelOutput((out ?? "").toString()); +} + +/** Resolve the claude binary once (PATH lookup), with the usual fallback. */ +export function resolveClaudeBin(claudeBin?: string): string { + return claudeBin ?? resolveCliBin("claude"); +} + +/** Build a GenerateFn (doc REFRESH) backed by the host `claude` CLI. */ export function makeClaudeGenerate(claudeBin?: string, timeoutMs = 120_000): GenerateFn { - const bin = claudeBin ?? resolveCliBin("claude"); - return async (ctx) => { - const prompt = buildRefreshPrompt(ctx); - const inv = buildClaudeInvocation(bin, prompt); - const out = execFileSync(inv.file, inv.args, { - ...inv.options, - encoding: "utf-8", - timeout: timeoutMs, - env: { ...process.env, HIVEMIND_WIKI_WORKER: "1", HIVEMIND_CAPTURE: "false" }, - }); - return unwrapModelOutput((out ?? "").toString()); - }; + const bin = resolveClaudeBin(claudeBin); + return async (ctx) => runClaudePrompt(bin, buildRefreshPrompt(ctx), timeoutMs); +} + +/** Build a GenerateDocFn (fresh doc GENERATION) backed by the host `claude` CLI. */ +export function makeClaudeGenerateDoc(claudeBin?: string, timeoutMs = 120_000): GenerateDocFn { + const bin = resolveClaudeBin(claudeBin); + return async (input) => runClaudePrompt(bin, buildGeneratePrompt(input), timeoutMs); } diff --git a/tests/shared/docs-generate.test.ts b/tests/shared/docs-generate.test.ts new file mode 100644 index 00000000..799f7a36 --- /dev/null +++ b/tests/shared/docs-generate.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +// Stub the read-stability gate to a single pass-through query (see docs.test.ts). +vi.mock("../../src/docs/stable-read.js", () => ({ + stableUnionRows: (q: (sql: string) => unknown, sql: string) => q(sql), +})); + +import { normalizeForHash, hashSource } from "../../src/docs/anchors.js"; +import { globToRegExp, selectTargets, generateDocs, DEFAULT_EXCLUDE_GLOBS } from "../../src/docs/generate.js"; +import type { GraphNode, GraphSnapshot } from "../../src/graph/types.js"; + +function node(id: string, file: string, loc: string, kind: GraphNode["kind"] = "function"): GraphNode { + return { id, label: id, kind, source_file: file, source_location: loc, language: "typescript", exported: true }; +} +function snap(nodes: GraphNode[]): GraphSnapshot { + return { nodes, links: [] } as unknown as GraphSnapshot; +} +function mockQuery(rowsPerCall: Array> = []) { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { calls.push(sql); return rowsPerCall; }); + return { calls, query }; +} + +// ── normalizeForHash (the false-positive fix) ───────────────────────────────── + +describe("normalizeForHash", () => { + it("ignores comments — a comment-only edit yields the SAME hash", () => { + const a = "function f() {\n // does a thing\n return 1;\n}"; + const b = "function f() {\n // does a completely different thing\n return 1;\n}"; + expect(hashSource(a)).toBe(hashSource(b)); + }); + it("ignores trailing whitespace and blank lines", () => { + const a = "function f() {\n return 1;\n}"; + const b = "function f() { \n\n return 1;\n\n}\n"; + expect(hashSource(a)).toBe(hashSource(b)); + }); + it("strips block comments too", () => { + expect(normalizeForHash("a();/* x */\nb();")).toBe("a();\nb();"); + }); + it("STILL detects a real code change (different identifier / literal)", () => { + expect(hashSource("return 1;")).not.toBe(hashSource("return 2;")); + expect(hashSource("const x = 1;")).not.toBe(hashSource("const y = 1;")); + }); + it("uses # comments for python and preserves indentation", () => { + expect(normalizeForHash("def f():\n # comment\n return 1", "python")).toBe("def f():\n return 1"); + }); +}); + +// ── globToRegExp ────────────────────────────────────────────────────────────── + +describe("globToRegExp", () => { + it("matches ** across directories and * within a segment", () => { + expect(globToRegExp("**/*.test.ts").test("src/a/b.test.ts")).toBe(true); + expect(globToRegExp("**/*.test.ts").test("src/b.ts")).toBe(false); + expect(globToRegExp("src/*.ts").test("src/a.ts")).toBe(true); + expect(globToRegExp("src/*.ts").test("src/a/b.ts")).toBe(false); + }); + it("escapes regex metacharacters and matches dotfiles", () => { + expect(globToRegExp("**/*.config.*").test("src/app.config.js")).toBe(true); + expect(globToRegExp("**/*.d.ts").test("types/x.d.ts")).toBe(true); + }); +}); + +// ── selectTargets ───────────────────────────────────────────────────────────── + +describe("selectTargets", () => { + const nodes = [ + node("src/a.ts:foo:function", "src/a.ts", "L1"), + node("src/a.ts:Bar:class", "src/a.ts", "L5", "class"), + node("src/a.test.ts:t:function", "src/a.test.ts", "L1"), // excluded by default + node("src/x.d.ts:T:type_alias", "src/x.d.ts", "L1", "type_alias"), // excluded by default + node("src/index.ts:re:function", "src/index.ts", "L1"), // barrel, excluded by default + node("src/b.ts:C:const", "src/b.ts", "L1", "const"), // non-documentable kind + ]; + it("file scope groups documentable symbols per file, default-excluding tests/d.ts/barrels", () => { + const t = selectTargets(snap(nodes), { scope: "file" }); + expect(t.map((x) => x.doc_id)).toEqual(["src/a.ts"]); // only a.ts survives + expect(t[0].symbols.map((s) => s.id).sort()).toEqual(["src/a.ts:Bar:class", "src/a.ts:foo:function"]); + }); + it("symbol scope yields one target per documentable symbol", () => { + const t = selectTargets(snap(nodes), { scope: "symbol" }); + expect(t.map((x) => x.doc_id).sort()).toEqual(["src/a.ts:Bar:class", "src/a.ts:foo:function"]); + }); + it("honors an explicit include filter", () => { + const t = selectTargets(snap(nodes), { scope: "file", include: ["src/b.ts"] }); + expect(t).toEqual([]); // b.ts only had a const (non-documentable) + }); + it("honors an additional exclude on top of the defaults", () => { + const t = selectTargets(snap(nodes), { scope: "file", exclude: ["src/a.ts"] }); + expect(t).toEqual([]); + }); + it("DEFAULT_EXCLUDE_GLOBS covers tests, d.ts, config and barrels", () => { + expect(DEFAULT_EXCLUDE_GLOBS.some((g) => globToRegExp(g).test("a/b.test.ts"))).toBe(true); + expect(DEFAULT_EXCLUDE_GLOBS.some((g) => globToRegExp(g).test("a/b.d.ts"))).toBe(true); + }); +}); + +// ── generateDocs (orchestrator) ─────────────────────────────────────────────── + +describe("generateDocs", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-gen-")); + writeFileSync(join(dir, "a.ts"), "export function foo() {\n return 1;\n}\n"); + writeFileSync(join(dir, "a.test.ts"), "export function t() {}\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + const s = () => snap([ + node("a.ts:foo:function", "a.ts", "L1-L3"), + node("a.test.ts:t:function", "a.test.ts", "L1"), + ]); + + it("creates a file-scope doc for non-excluded files, anchored to their symbols", async () => { + const { calls, query } = mockQuery([]); // setDoc: SELECT [] then INSERT v1 + const generate = vi.fn(async () => "# a.ts\nfoo returns 1."); + const report = await generateDocs({ + query, tableName: "hivemind_docs", snap: s(), repoRoot: dir, + project: "p", existing: new Set(), generate, concurrency: 2, + }); + expect(report.created).toBe(1); // a.test.ts excluded by default + expect(report.targets).toBe(1); + expect(generate).toHaveBeenCalledOnce(); + const insert = calls.find((c) => /INSERT INTO "hivemind_docs"/.test(c))!; + expect(insert).toContain("a.ts:foo:function"); // anchored + expect(insert).toContain("foo returns 1."); + }); + + it("skips files that already have a doc (idempotent) unless --force", async () => { + const { query } = mockQuery([]); + const generate = vi.fn(async () => "x"); + const report = await generateDocs({ + query, tableName: "hivemind_docs", snap: s(), repoRoot: dir, + existing: new Set(["a.ts"]), generate, + }); + expect(report.created).toBe(0); + expect(generate).not.toHaveBeenCalled(); + }); + + it("records a failure when generation throws", async () => { + const { query } = mockQuery([]); + const report = await generateDocs({ + query, tableName: "hivemind_docs", snap: s(), repoRoot: dir, + existing: new Set(), generate: async () => { throw new Error("LLM down"); }, + }); + expect(report.failed).toBe(1); + expect(report.outcomes[0].reason).toMatch(/LLM down/); + }); +}); From 4a961e7d54f948a57a7ec78411278488f3f26d85 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 10:48:14 +0000 Subject: [PATCH 21/39] fix: retry timed-out doc writes so bulk generate survives backend load --- src/docs/generate.ts | 20 +++++++++--- src/docs/index.ts | 4 +-- src/docs/write.ts | 57 ++++++++++++++++++++++++++++++++ tests/shared/docs.test.ts | 68 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/src/docs/generate.ts b/src/docs/generate.ts index 29380238..30d9e52b 100644 --- a/src/docs/generate.ts +++ b/src/docs/generate.ts @@ -17,7 +17,7 @@ */ import { buildAnchor, readSymbolSource } from "./anchors.js"; -import { setDoc } from "./write.js"; +import { insertDocResilient, setDoc } from "./write.js"; import type { DocAnchor, QueryFn } from "./read.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -218,16 +218,28 @@ export async function generateDocs(args: GenerateArgs): Promise { return; } try { - await setDoc(args.query, args.tableName, { + const write = { doc_id: t.doc_id, path: defaultVfsPath(project, t.doc_id), content, anchors, - tier: "fast", + tier: "fast" as const, project, agent: args.agent ?? "docs-generate", plugin_version: args.pluginVersion, - }); + }; + // Fresh docs (the common bulk-generate case) go through insertDocResilient + // — a single INSERT, NOT setDoc's read-stability gate (~N reads). Under + // high concurrency the gate's read storm overloaded the backend and timed + // writes out. The resilient variant retries the INSERT on the backend's + // intermittent under-load timeouts (checking whether the row already + // landed before each retry, so it never forks a second v1). Only + // re-authored (--force) docs need setDoc's version-bump. + if (args.force && args.existing.has(t.doc_id)) { + await setDoc(args.query, args.tableName, write); + } else { + await insertDocResilient(args.query, args.tableName, write); + } outcomes.push({ doc_id: t.doc_id, status: "created" }); } catch (err) { outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `write failed: ${(err as Error).message}` }); diff --git a/src/docs/index.ts b/src/docs/index.ts index ae78e892..98623834 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -6,8 +6,8 @@ * change for callers. */ -export { insertDoc, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; -export type { InsertDocInput, EditDocInput, SetDocInput, WriteResult } from "./write.js"; +export { insertDoc, insertDocResilient, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; +export type { InsertDocInput, EditDocInput, SetDocInput, WriteResult, ResilientWriteOpts } from "./write.js"; export { listDocs, getDocLatest, parseAnchors } from "./read.js"; export type { diff --git a/src/docs/write.ts b/src/docs/write.ts index 0c1cad40..67316f1e 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -147,6 +147,63 @@ export async function insertDoc( return { doc_id: input.doc_id, version: 1 }; } +/** Default retry budget + backoff (ms) for a timed-out INSERT. */ +const WRITE_RETRIES = 3; +const WRITE_BACKOFF_MS = [500, 1500, 4000]; + +function isTimeoutError(err: unknown): boolean { + return err instanceof Error && /timeout/i.test(err.message); +} + +export interface ResilientWriteOpts { + /** Max retries after the first attempt. Default 3. */ + retries?: number; + /** Backoff schedule in ms, indexed by retry number. */ + backoffMs?: number[]; + /** Injectable sleep (tests). Default real setTimeout. */ + sleep?: (ms: number) => Promise; +} + +/** + * `insertDoc` hardened against the Deeplake backend's under-load write + * timeouts. A write can abort client-side at the 10s query timeout while the + * backend actually committed the row — so on each retry, before re-inserting, + * we read back the latest version (`getDocLatest`): if the row already landed + * we return it instead of INSERTing again, which would otherwise fork history + * into two parallel `version=1` rows. Only timeouts are retried; any other + * error surfaces immediately. + * + * This is what makes bulk `docs generate` reliable on a real codebase: the + * backend intermittently times out individual writes under concurrency, and a + * naive single INSERT drops those docs on the floor (18/33 in one real run). + */ +export async function insertDocResilient( + query: QueryFn, + tableName: string, + input: InsertDocInput, + opts: ResilientWriteOpts = {}, +): Promise { + const retries = opts.retries ?? WRITE_RETRIES; + const backoff = opts.backoffMs ?? WRITE_BACKOFF_MS; + const sleep = opts.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + let lastErr: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await insertDoc(query, tableName, input); + } catch (err) { + if (!isTimeoutError(err)) throw err; + lastErr = err; + if (attempt === retries) break; + await sleep(backoff[Math.min(attempt, backoff.length - 1)]); + // Did the timed-out attempt actually commit server-side? If so, stop — + // re-inserting would fork a second version=1 row. + const landed = await getDocLatest(query, tableName, input.doc_id).catch(() => null); + if (landed) return { doc_id: landed.doc_id, version: landed.version }; + } + } + throw lastErr ?? new Error("insertDocResilient: exhausted retries"); +} + /** * Edit an existing doc. Reads the latest version, then INSERTs a new row * with version+1 carrying the merged fields (omitted fields inherit from diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 3d7078b9..721721a5 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -9,6 +9,7 @@ vi.mock("../../src/docs/stable-read.js", () => ({ })); import { insertDoc, + insertDocResilient, editDoc, setDoc, archiveDoc, @@ -167,6 +168,73 @@ describe("insertDoc", () => { }); }); +// ── insertDocResilient (timeout-safe write — the bulk-generate reliability fix) ─ + +describe("insertDocResilient", () => { + const noSleep = async () => {}; + const timeout = () => { + throw new Error("Query timeout after 10000ms"); + }; + + it("retries a timed-out INSERT and succeeds on the next attempt", async () => { + // INSERT times out → read-back finds nothing → INSERT again → ok. + const { calls, query } = mockQuery([timeout, () => [], () => []]); + const result = await insertDocResilient( + query, + TBL, + { doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "x" }, + { sleep: noSleep }, + ); + expect(result).toEqual({ doc_id: "a.ts", version: 1 }); + expect(calls).toHaveLength(3); + expect(calls[0]).toMatch(/^INSERT INTO/); + expect(calls[1]).toMatch(/^SELECT/); // the landed-check read-back + expect(calls[2]).toMatch(/^INSERT INTO/); + }); + + it("does NOT re-insert when the timed-out write actually landed (no forked v1)", async () => { + // INSERT times out client-side but committed server-side → read-back finds + // the row → return it, issue no second INSERT. + const { calls, query } = mockQuery([timeout, () => [fakeRow({ doc_id: "a.ts", version: 1 })]]); + const result = await insertDocResilient( + query, + TBL, + { doc_id: "a.ts", path: "/docs/p/a.ts.md", content: "x" }, + { sleep: noSleep }, + ); + expect(result).toEqual({ doc_id: "a.ts", version: 1 }); + expect(calls).toHaveLength(2); + expect(calls.filter(c => c.startsWith("INSERT INTO"))).toHaveLength(1); + }); + + it("surfaces a non-timeout error immediately without retrying", async () => { + const { calls, query } = mockQuery([ + () => { + throw new Error("Query failed: 403: forbidden"); + }, + ]); + await expect( + insertDocResilient(query, TBL, { doc_id: "a.ts", path: "/p", content: "x" }, { sleep: noSleep }), + ).rejects.toThrow(/403/); + expect(calls).toHaveLength(1); + }); + + it("gives up after the retry budget and throws the timeout", async () => { + // Every INSERT times out, read-back never finds the row. + const { calls, query } = mockQuery([timeout, () => [], timeout, () => [], timeout]); + await expect( + insertDocResilient( + query, + TBL, + { doc_id: "a.ts", path: "/p", content: "x" }, + { retries: 2, sleep: noSleep }, + ), + ).rejects.toThrow(/timeout/); + // 3 INSERT attempts (retries=2) + 2 read-backs between them. + expect(calls.filter(c => c.startsWith("INSERT INTO"))).toHaveLength(3); + }); +}); + // ── editDoc ─────────────────────────────────────────────────────────────────── describe("editDoc", () => { From c32ae51577b0e88215cc46316d92ff081b192a63 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 12:31:04 +0000 Subject: [PATCH 22/39] fix: archive docs for deleted or renamed files instead of rewriting --- src/commands/docs.ts | 5 ++++- src/docs/refresh.ts | 33 +++++++++++++++++++++++++++---- tests/shared/docs-refresh.test.ts | 27 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index d8902504..1fe93f8f 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -360,9 +360,12 @@ export async function runDocsCommand(args: string[]): Promise { agent: cfg.userName, pluginVersion, }); - console.log(`Refreshed ${report.refreshed}, rejected ${report.rejected}, skipped ${report.skipped}.`); + console.log( + `Refreshed ${report.refreshed}, archived ${report.archived}, rejected ${report.rejected}, skipped ${report.skipped}.`, + ); for (const o of report.outcomes) { if (o.status === "refreshed") console.log(` refreshed ${o.doc_id} → v${o.version}`); + else if (o.status === "archived") console.log(` archived ${o.doc_id} → v${o.version} (${(o.reasons ?? []).join("; ")})`); else console.log(` ${o.status} ${o.doc_id}: ${(o.reasons ?? []).join("; ")}`); } return; diff --git a/src/docs/refresh.ts b/src/docs/refresh.ts index bd844144..f20ec47c 100644 --- a/src/docs/refresh.ts +++ b/src/docs/refresh.ts @@ -18,7 +18,7 @@ import { buildAnchor, readSymbolSource } from "./anchors.js"; import { gateDocEdit, type GateResult } from "./gate.js"; -import { setDoc } from "./write.js"; +import { archiveDoc, setDoc } from "./write.js"; import type { DocAnchor, DocRow, QueryFn } from "./read.js"; import type { ImpactedDoc, StaleReason } from "./impact.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -42,10 +42,10 @@ export type GenerateFn = (ctx: RefreshContext) => Promise; export interface RefreshOutcome { doc_id: string; - status: "refreshed" | "rejected" | "skipped"; - /** New version number when refreshed. */ + status: "refreshed" | "rejected" | "skipped" | "archived"; + /** New version number when refreshed or archived. */ version?: number; - /** Gate rejection reasons, or skip explanation. */ + /** Gate rejection reasons, or skip/archive explanation. */ reasons?: string[]; } @@ -54,6 +54,7 @@ export interface RefreshReport { refreshed: number; rejected: number; skipped: number; + archived: number; } export interface RefreshArgs { @@ -154,6 +155,29 @@ export async function refreshDocs(args: RefreshArgs): Promise { } const newAnchors = reanchor(doc, nodeById, args.repoRoot); + + // Fully-orphaned doc: it HAD anchors, and every one of them vanished from + // the graph (the documented file was deleted or renamed). Re-authoring it + // would burn an LLM call and produce an anchor-less zombie that drift + // detection can never flag again. Archive it instead — soft delete, + // audit trail preserved — and spend no token. A PARTIAL orphan (the file + // still exists, only some symbols removed) keeps ≥1 anchor and falls + // through to a normal refresh below. + if (doc.anchors.length > 0 && newAnchors.length === 0) { + const res = await archiveDoc(args.query, args.tableName, { + doc_id: doc.doc_id, + agent: args.agent ?? "docs-refresh", + plugin_version: args.pluginVersion, + }); + outcomes.push({ + doc_id: imp.doc_id, + status: "archived", + version: res.version, + reasons: ["all anchored symbols gone (file deleted/renamed)"], + }); + continue; + } + const changedSymbols = gatherChangedSymbols(imp.reasons, nodeById, args.repoRoot); let newContent: string; @@ -195,5 +219,6 @@ export async function refreshDocs(args: RefreshArgs): Promise { refreshed: outcomes.filter((o) => o.status === "refreshed").length, rejected: outcomes.filter((o) => o.status === "rejected").length, skipped: outcomes.filter((o) => o.status === "skipped").length, + archived: outcomes.filter((o) => o.status === "archived").length, }; } diff --git a/tests/shared/docs-refresh.test.ts b/tests/shared/docs-refresh.test.ts index db4cd7bb..df01a2f4 100644 --- a/tests/shared/docs-refresh.test.ts +++ b/tests/shared/docs-refresh.test.ts @@ -224,6 +224,33 @@ describe("refreshDocs", () => { expect(calls).toHaveLength(0); }); + it("archives a fully-orphaned doc (file deleted/renamed) WITHOUT calling the LLM", async () => { + // doc anchored ONLY to a symbol that's gone; the snapshot has none of it → + // the documented file was deleted or renamed. Archive, don't re-author. + const gone = "a.ts:gone:function"; + const d = doc({ anchors: [{ symbol_id: gone, content_hash: "x" }] }); + const emptySnap = snap([]); + const { calls, query } = mockQuery([ + () => [{ id: "r", doc_id: "a.ts", version: 3, content: "old", anchors: "[]", tier: "fast", status: "active", project: "p", created_at: "t", updated_at: "t" }], // getDocLatest in archiveDoc + () => [], // INSERT of the archived version + ]); + const generate = vi.fn(async () => "should never be called"); + const report = await refreshDocs({ + query, tableName: "hivemind_docs", snap: emptySnap, repoRoot: dir, + impacted: [{ doc_id: "a.ts", reasons: [{ kind: "symbol_missing", symbol_id: gone }] }], + docsById: new Map([["a.ts", d]]), generate, + }); + expect(report.archived).toBe(1); + expect(report.refreshed).toBe(0); + expect(report.outcomes[0]).toMatchObject({ doc_id: "a.ts", status: "archived", version: 4 }); + // No token spent on a deleted file. + expect(generate).not.toHaveBeenCalled(); + // archiveDoc = getDocLatest + INSERT(status='archived'); nothing else. + expect(calls).toHaveLength(2); + expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + expect(calls[1]).toContain("'archived'"); + }); + it("drops a dangling anchor when its symbol vanished from the graph", async () => { // doc anchored to foo + a gone symbol; snapshot only has foo. const d = doc({ anchors: [{ symbol_id: foo.id, content_hash: "x" }, { symbol_id: "a.ts:gone:function", content_hash: "y" }] }); From 2d2d2ce745aa8820dbb55cf1b1faab8774b77485 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 14:09:44 +0000 Subject: [PATCH 23/39] feat: add browsable per-directory docs index --- src/commands/docs.ts | 44 ++++++++ src/docs/index-render.ts | 149 +++++++++++++++++++++++++ src/docs/index.ts | 6 +- src/docs/read.ts | 91 ++++++++++++++- tests/shared/docs-index-render.test.ts | 81 ++++++++++++++ tests/shared/docs.test.ts | 71 ++++++++++++ 6 files changed, 440 insertions(+), 2 deletions(-) create mode 100644 src/docs/index-render.ts create mode 100644 tests/shared/docs-index-render.test.ts diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 1fe93f8f..64b193e9 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -31,11 +31,17 @@ import { setDoc, archiveDoc, listDocs, + listDocMeta, + listDocsByIds, getDocLatest, computeImpactedDocs, refreshDocs, buildAnchor, + buildDocsIndex, + dirOf, + firstDocLine, type DocRow, + type DocMeta, type DocTier, type DocAnchor, } from "../docs/index.js"; @@ -50,6 +56,9 @@ hivemind docs — per-file documentation kept fresh on code deltas Usage: hivemind docs set [""] [--file ] [--project P] [--tier fast|slow] [--path ] hivemind docs show + hivemind docs index [] + Browsable per-directory index of the docs (metadata only). With no + argument shows the top level; pass a directory to drill in. hivemind docs list [--project P] [--status active|archived|all] [--limit N] hivemind docs archive hivemind docs refresh [--cwd ] [--dry-run] @@ -253,6 +262,41 @@ export async function runDocsCommand(args: string[]): Promise { return; } + if (sub === "index") { + // Browsable per-directory index. `atDir` ("" = root) is an optional + // positional; drilling in scopes the metadata read to that subtree. + const atDir = (stripKnownFlags(args.slice(1))[0] ?? "").replace(/\/+$/, ""); + let meta: DocMeta[] = []; + try { + const rows = await listDocMeta(query, tableName, { dirPrefix: atDir }); + meta = rows.map((r) => ({ + doc_id: r.doc_id, + version: r.version, + updated_at: r.updated_at, + status: r.status, + tier: r.tier, + })); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + // Fetch content ONLY for files directly in this directory → 1-line summaries. + const directFiles = meta + .filter((m) => m.status === "active" && dirOf(m.doc_id) === atDir) + .map((m) => m.doc_id); + const summaries = new Map(); + if (directFiles.length > 0) { + try { + for (const d of await listDocsByIds(query, tableName, directFiles)) { + summaries.set(d.doc_id, firstDocLine(d.content)); + } + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + } + } + console.log(buildDocsIndex(meta, atDir, summaries)); + return; + } + if (sub === "show") { const docId = stripKnownFlags(args.slice(1))[0]; if (!docId) { diff --git a/src/docs/index-render.ts b/src/docs/index-render.ts new file mode 100644 index 00000000..b5b19d1f --- /dev/null +++ b/src/docs/index-render.ts @@ -0,0 +1,149 @@ +/** + * Pure renderer for the per-directory docs index (browsable like the memory + * `index.md`, but a SEPARATE surface under the code side, not mixed into the + * session-memory index — see the design note in src/commands/docs.ts). + * + * The index is a filesystem-shaped drill-down so it stays small on a large + * repo: at any directory you see (a) the immediate SUB-directories that hold + * docs, with aggregate counts, and (b) the docs for files directly in this + * directory. You never render one flat list of every file. + * + * Levels 0/N-with-subdirs need only METADATA (doc_id, version, updated_at) — + * no content is pulled. The optional `summaries` map is filled only for the + * files directly in the viewed directory (a bounded, cheap content fetch), so + * the "1 line per file" description never costs a full-table content read. + * + * Pure and synchronous: all I/O (the metadata query, the summary fetch) is the + * caller's job, which keeps this exhaustively unit-testable. + */ + +/** Light per-doc metadata — everything the index needs except content. */ +export interface DocMeta { + /** Documented source file path, e.g. `src/graph/diff.ts`. The stable key. */ + doc_id: string; + version: number; + updated_at: string; + status: string; + tier: string; +} + +/** The directory portion of a `doc_id` ("" for a top-level file). */ +export function dirOf(docId: string): string { + const i = docId.lastIndexOf("/"); + return i < 0 ? "" : docId.slice(0, i); +} + +/** The immediate child of `atDir` on the path to `docId`, or null. */ +function childUnder(atDir: string, docId: string): { kind: "dir" | "file"; name: string } | null { + const prefix = atDir === "" ? "" : atDir + "/"; + if (!docId.startsWith(prefix)) return null; + const rest = docId.slice(prefix.length); + if (rest === "" || rest.startsWith("/")) return null; + const slash = rest.indexOf("/"); + if (slash < 0) return { kind: "file", name: rest }; // a file directly in atDir + return { kind: "dir", name: rest.slice(0, slash) }; // an immediate subdirectory +} + +/** First meaningful line of a doc body, for the per-file summary column. */ +export function firstDocLine(content: string, max = 90): string { + for (const raw of content.split("\n")) { + const line = raw.trim(); + if (line === "" || line.startsWith("#") || line === "---") continue; + return line.length > max ? line.slice(0, max - 1).trimEnd() + "…" : line; + } + return ""; +} + +function dateOnly(ts: string): string { + return ts.slice(0, 10); +} + +interface DirAgg { + count: number; + latest: string; +} + +/** + * Render the index for directory `atDir` ("" = root) from the full metadata + * list. `summaries` (doc_id → one-liner) is optional and used only for the + * files shown at this level. + */ +export function buildDocsIndex( + meta: DocMeta[], + atDir = "", + summaries: Map = new Map(), +): string { + const active = meta.filter((m) => m.status === "active"); + + const subdirs = new Map(); + const files: DocMeta[] = []; + for (const m of active) { + const child = childUnder(atDir, m.doc_id); + if (!child) continue; + if (child.kind === "file") { + files.push(m); + } else { + const agg = subdirs.get(child.name); + if (agg) { + agg.count++; + if (m.updated_at > agg.latest) agg.latest = m.updated_at; + } else { + subdirs.set(child.name, { count: 1, latest: m.updated_at }); + } + } + } + + const title = atDir === "" ? "# Docs Index" : `# Docs: ${atDir}/`; + const lines: string[] = [ + title, + "", + "Per-file documentation, kept fresh on code changes. Drill into a directory,", + "or open a file's doc directly. Metadata only — open a leaf for the content.", + "", + ]; + + if (subdirs.size > 0) { + lines.push("## Directories", ""); + lines.push("| Directory | Docs | Last updated |"); + lines.push("|-----------|------|--------------|"); + for (const name of [...subdirs.keys()].sort()) { + const agg = subdirs.get(name)!; + const rel = `${name}/index.md`; + lines.push(`| [${name}/](${rel}) | ${agg.count} | ${dateOnly(agg.latest)} |`); + } + lines.push(""); + } + + if (files.length > 0) { + lines.push("## Files", ""); + const hasSummary = files.some((f) => (summaries.get(f.doc_id) ?? "") !== ""); + if (hasSummary) { + lines.push("| File | Version | Updated | Summary |"); + lines.push("|------|---------|---------|---------|"); + } else { + lines.push("| File | Version | Updated |"); + lines.push("|------|---------|---------|"); + } + for (const f of files.sort((a, b) => a.doc_id.localeCompare(b.doc_id))) { + const base = f.doc_id.slice(f.doc_id.lastIndexOf("/") + 1); + const rel = `${base}.md`; + const ver = `v${f.version}`; + if (hasSummary) { + lines.push(`| [${base}](${rel}) | ${ver} | ${dateOnly(f.updated_at)} | ${summaries.get(f.doc_id) ?? ""} |`); + } else { + lines.push(`| [${base}](${rel}) | ${ver} | ${dateOnly(f.updated_at)} |`); + } + } + lines.push(""); + } + + if (subdirs.size === 0 && files.length === 0) { + lines.push(atDir === "" ? "_(no docs yet — run `hivemind docs generate`)_" : `_(no docs under ${atDir}/)_`); + lines.push(""); + } + + const totalActive = active.length; + const archived = meta.length - totalActive; + lines.push("---", `${totalActive} active doc(s)${archived > 0 ? `, ${archived} archived` : ""}.`); + return lines.join("\n"); +} diff --git a/src/docs/index.ts b/src/docs/index.ts index 98623834..c0133d61 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -9,15 +9,19 @@ export { insertDoc, insertDocResilient, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; export type { InsertDocInput, EditDocInput, SetDocInput, WriteResult, ResilientWriteOpts } from "./write.js"; -export { listDocs, getDocLatest, parseAnchors } from "./read.js"; +export { listDocs, listDocMeta, listDocsByIds, getDocLatest, parseAnchors } from "./read.js"; export type { DocRow, + DocMetaRow, DocAnchor, DocTier, ListDocsOpts, QueryFn, } from "./read.js"; +export { buildDocsIndex, dirOf, firstDocLine } from "./index-render.js"; +export type { DocMeta } from "./index-render.js"; + export { parseSourceLocation, readSymbolSource, diff --git a/src/docs/read.ts b/src/docs/read.ts index a5e14d70..4f21af7e 100644 --- a/src/docs/read.ts +++ b/src/docs/read.ts @@ -13,7 +13,7 @@ * rather than throwing, so a single bad row never poisons a list read. */ -import { sqlIdent, sqlStr } from "../utils/sql.js"; +import { sqlIdent, sqlLike, sqlStr } from "../utils/sql.js"; import { stableUnionRows } from "./stable-read.js"; export type QueryFn = (sql: string) => Promise>>; @@ -102,6 +102,95 @@ export async function listDocs( return filtered.slice(0, opts.limit ?? 200); } +/** Light per-doc metadata for the index — no content, no anchors. */ +export interface DocMetaRow { + doc_id: string; + version: number; + updated_at: string; + status: string; + tier: DocTier; +} + +/** + * Latest-version METADATA for every doc (optionally scoped to a directory + * prefix), WITHOUT pulling the `content`/`anchors` columns. This is the cheap + * read behind the browsable docs index: the top levels need only counts and + * timestamps, so we never transfer document bodies to render them. + * + * `dirPrefix` (e.g. `src/graph`) filters server-side via `doc_id LIKE`, so a + * drill-down into one directory reads only that directory's rows. + */ +export async function listDocMeta( + query: QueryFn, + tableName: string, + opts: { dirPrefix?: string } = {}, +): Promise { + const safe = sqlIdent(tableName); + const where = + opts.dirPrefix !== undefined && opts.dirPrefix !== "" + ? ` WHERE doc_id LIKE '${sqlLike(opts.dirPrefix)}/%'` + : ""; + // `id` is selected (not returned) so the read-stability gate can union rows + // by their unique key — see stableUnionRows(idKey="id"). + const rows = await stableUnionRows( + query, + `SELECT id, doc_id, version, updated_at, status, tier FROM "${safe}"${where}`, + ); + const latest = new Map(); + for (const r of rows) { + const doc_id = String(r.doc_id ?? ""); + if (doc_id === "") continue; + const vRaw = r.version; + const version = typeof vRaw === "number" ? vRaw : Number(vRaw); + if (!Number.isFinite(version)) continue; + const updated_at = String(r.updated_at ?? ""); + const prev = latest.get(doc_id); + if (!prev || version > prev.version || (version === prev.version && updated_at > prev.updated_at)) { + const tier = String(r.tier ?? "fast"); + latest.set(doc_id, { + doc_id, + version, + updated_at, + status: String(r.status ?? ""), + tier: tier === "slow" ? "slow" : "fast", + }); + } + } + return [...latest.values()]; +} + +/** + * Latest full row for each of `docIds` (a filtered `doc_id IN (...)` read). + * Two uses: fetching the small set of files' content for the index summary + * column, and — the scale path — loading only the docs a commit's diff can + * affect instead of the whole table. Returns latest-per-doc, active or not; + * the caller filters by status. An empty `docIds` returns `[]` with no query. + */ +export async function listDocsByIds( + query: QueryFn, + tableName: string, + docIds: string[], +): Promise { + const ids = [...new Set(docIds.filter((d) => d !== ""))]; + if (ids.length === 0) return []; + const safe = sqlIdent(tableName); + const inList = ids.map((d) => `'${sqlStr(d)}'`).join(", "); + const rows = await stableUnionRows( + query, + `SELECT ${SELECT_COLS} FROM "${safe}" WHERE doc_id IN (${inList})`, + ); + const latest = new Map(); + for (const r of rows) { + const row = normalize(r); + if (!row) continue; + const prev = latest.get(row.doc_id); + if (!prev || row.version > prev.version || (row.version === prev.version && row.updated_at > prev.updated_at)) { + latest.set(row.doc_id, row); + } + } + return [...latest.values()]; +} + /** * Return the latest version of a single doc by `doc_id`, or `null` if it * does not exist. Used by `editDoc` in ./write.ts to carry over the prior diff --git a/tests/shared/docs-index-render.test.ts b/tests/shared/docs-index-render.test.ts new file mode 100644 index 00000000..eb161e1c --- /dev/null +++ b/tests/shared/docs-index-render.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { buildDocsIndex, dirOf, firstDocLine, type DocMeta } from "../../src/docs/index-render.js"; + +function m(doc_id: string, over: Partial = {}): DocMeta { + return { doc_id, version: 1, updated_at: "2026-07-01T10:00:00.000Z", status: "active", tier: "fast", ...over }; +} + +describe("dirOf", () => { + it("returns the parent directory, or '' for a top-level file", () => { + expect(dirOf("src/graph/diff.ts")).toBe("src/graph"); + expect(dirOf("index.ts")).toBe(""); + }); +}); + +describe("firstDocLine", () => { + it("skips the leading heading, blank lines, and the --- separator", () => { + expect(firstDocLine("# Title\n\n---\nThe real summary line.\nmore")).toBe("The real summary line."); + }); + it("truncates long lines with an ellipsis", () => { + expect(firstDocLine("x".repeat(200), 20)).toMatch(/…$/); + expect(firstDocLine("x".repeat(200), 20).length).toBeLessThanOrEqual(20); + }); + it("returns empty string when there is no prose", () => { + expect(firstDocLine("# Only\n## Headings")).toBe(""); + }); +}); + +describe("buildDocsIndex", () => { + const corpus: DocMeta[] = [ + m("src/graph/diff.ts", { version: 2 }), + m("src/graph/cache.ts"), + m("src/graph/extract/python.ts"), + m("src/graph/extract/rust.ts"), + m("src/docs/read.ts"), + m("README.md"), // top-level file + m("src/graph/gone.ts", { status: "archived" }), + ]; + + it("root groups by immediate subdirectory and lists top-level files, not the whole tree", () => { + const out = buildDocsIndex(corpus, ""); + // Immediate children of root: the `src` directory + the README file. + expect(out).toContain("[src/](src/index.md)"); + expect(out).toContain("[README.md](README.md.md)"); + // It must NOT flatten deep files into the root view. + expect(out).not.toContain("diff.ts"); + expect(out).not.toContain("python.ts"); + }); + + it("drilling into a directory shows its immediate subdirs AND its direct files", () => { + const out = buildDocsIndex(corpus, "src/graph"); + expect(out).toContain("# Docs: src/graph/"); + expect(out).toContain("[extract/](extract/index.md)"); // subdir with 2 docs + expect(out).toContain("| [extract/](extract/index.md) | 2 |"); + expect(out).toContain("[diff.ts](diff.ts.md)"); // direct file + expect(out).toContain("[cache.ts](cache.ts.md)"); + expect(out).toContain("v2"); // diff.ts is version 2 + // extract/*.ts are under a subdir, not direct files here + expect(out).not.toContain("[python.ts]"); + }); + + it("shows the Summary column only when a summary is provided", () => { + const withSummary = buildDocsIndex(corpus, "src/docs", new Map([["src/docs/read.ts", "Read helpers."]])); + expect(withSummary).toContain("| File | Version | Updated | Summary |"); + expect(withSummary).toContain("Read helpers."); + const without = buildDocsIndex(corpus, "src/docs"); + expect(without).toContain("| File | Version | Updated |"); + expect(without).not.toContain("Summary"); + }); + + it("excludes archived docs from the tree but counts them in the footer", () => { + const out = buildDocsIndex(corpus, "src/graph"); + expect(out).not.toContain("gone.ts"); // archived → not shown + // 6 active total across the corpus, 1 archived. + expect(out).toMatch(/6 active doc\(s\), 1 archived\./); + }); + + it("renders an empty-state line when a directory has no docs", () => { + expect(buildDocsIndex([], "")).toContain("no docs yet"); + expect(buildDocsIndex(corpus, "src/nowhere")).toContain("no docs under src/nowhere/"); + }); +}); diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 721721a5..29e36ca3 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -14,6 +14,8 @@ import { setDoc, archiveDoc, listDocs, + listDocMeta, + listDocsByIds, getDocLatest, parseAnchors, _MAX_CONTENT_LENGTH, @@ -494,6 +496,75 @@ describe("getDocLatest", () => { }); }); +// ── listDocMeta (light index read — no content) ────────────────────────────── + +describe("listDocMeta", () => { + it("selects id (for the union key) but NOT content/anchors, dedups latest per doc", async () => { + const { calls, query } = mockQuery([ + () => [ + fakeRow({ id: "r1", doc_id: "a.ts", version: 1 }), + fakeRow({ id: "r2", doc_id: "a.ts", version: 2 }), + fakeRow({ id: "r3", doc_id: "b.ts", version: 1 }), + ], + ]); + const meta = await listDocMeta(query, TBL); + // The query must be metadata-only: id for the stability-gate union key, + // never the heavy content/anchors columns. + expect(calls[0]).toMatch(/SELECT id, doc_id, version, updated_at, status, tier/); + expect(calls[0]).not.toContain("content"); + expect(calls[0]).not.toContain("anchors"); + // latest-per-doc: a.ts collapses to v2, b.ts stays v1. + const byId = new Map(meta.map((r) => [r.doc_id, r.version])); + expect(byId.get("a.ts")).toBe(2); + expect(byId.get("b.ts")).toBe(1); + expect(meta).toHaveLength(2); + }); + + it("scopes to a directory prefix with an escaped LIKE", async () => { + const { calls, query } = mockQuery([() => []]); + await listDocMeta(query, TBL, { dirPrefix: "src/graph" }); + expect(calls[0]).toContain(`WHERE doc_id LIKE 'src/graph/%'`); + }); + + it("omits the WHERE clause when no dirPrefix is given", async () => { + const { calls, query } = mockQuery([() => []]); + await listDocMeta(query, TBL); + expect(calls[0]).not.toContain("WHERE"); + }); +}); + +// ── listDocsByIds (filtered read for index summaries + the scale path) ──────── + +describe("listDocsByIds", () => { + it("builds a de-duplicated IN list and returns latest-per-doc", async () => { + const { calls, query } = mockQuery([ + () => [ + fakeRow({ id: "r1", doc_id: "a.ts", version: 1, content: "old" }), + fakeRow({ id: "r2", doc_id: "a.ts", version: 3, content: "new" }), + fakeRow({ id: "r3", doc_id: "b.ts", version: 1, content: "b" }), + ], + ]); + const rows = await listDocsByIds(query, TBL, ["a.ts", "b.ts", "a.ts"]); + expect(calls[0]).toContain(`doc_id IN ('a.ts', 'b.ts')`); // deduped + const a = rows.find((r) => r.doc_id === "a.ts"); + expect(a?.version).toBe(3); + expect(a?.content).toBe("new"); + expect(rows).toHaveLength(2); + }); + + it("short-circuits to [] with no query on empty input", async () => { + const { calls, query } = mockQuery([]); + expect(await listDocsByIds(query, TBL, [])).toEqual([]); + expect(calls).toHaveLength(0); + }); + + it("escapes doc_ids in the IN list", async () => { + const { calls, query } = mockQuery([() => []]); + await listDocsByIds(query, TBL, ["x' OR '1'='1"]); + expect(calls[0]).toContain(`'x'' OR ''1''=''1'`); + }); +}); + // ── parseAnchors ────────────────────────────────────────────────────────────── describe("parseAnchors", () => { From 2b1a2c4481386ac817ad2aa33295b169044120eb Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 14:22:28 +0000 Subject: [PATCH 24/39] feat: serve the docs index as a virtual filesystem under docs/ --- src/docs/vfs-handler.ts | 72 ++++++++++++++++ src/hooks/pre-tool-use.ts | 28 +++++++ .../claude-code/pre-tool-use-branches.test.ts | 35 ++++++++ tests/shared/docs-vfs-handler.test.ts | 82 +++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 src/docs/vfs-handler.ts create mode 100644 tests/shared/docs-vfs-handler.test.ts diff --git a/src/docs/vfs-handler.ts b/src/docs/vfs-handler.ts new file mode 100644 index 00000000..6cb0eb24 --- /dev/null +++ b/src/docs/vfs-handler.ts @@ -0,0 +1,72 @@ +/** + * Virtual filesystem handler for the docs index under + * ~/.deeplake/memory/docs/ — the same intercept pattern as the graph mount + * (src/graph/vfs-handler.ts), but a SEPARATE surface from the session-memory + * index so browsing docs never pollutes memory retrieval. + * + * Unlike the graph handler (sync, reads a local snapshot), docs live in the + * Deeplake table, so this handler is ASYNC and takes a `query` seam. It stays + * cheap by construction: directory levels read metadata only (`listDocMeta`, + * no content), and one-line summaries are fetched only for the files directly + * in the viewed directory (`listDocsByIds`). + * + * Path shapes (relative to /docs/): + * "" | "index.md" → root index (top directories, counts) + * "" | "/index.md" → directory index (subdirs + files + summaries) + * ".md" → the doc content for source file + * + * `index.md` at any level is always the DIRECTORY index; a real file's doc is + * ".md" (e.g. `diff.ts.md`), so there is no collision. + */ + +import { listDocMeta, listDocsByIds, getDocLatest, type QueryFn } from "./read.js"; +import { buildDocsIndex, dirOf, firstDocLine, type DocMeta } from "./index-render.js"; + +export type DocsVfsResult = + | { kind: "ok"; body: string } + | { kind: "not-found"; message: string }; + +/** Resolve a `/docs/` subpath to rendered text from the docs table. */ +export async function handleDocsVfs( + subpath: string, + query: QueryFn, + tableName: string, +): Promise { + const path = subpath.replace(/^\/+/, "").replace(/\/+$/, ""); + + // Decide whether this is a directory index or a leaf doc. + let dir: string | null = null; + if (path === "" || path === "index.md") dir = ""; + else if (path.endsWith("/index.md")) dir = path.slice(0, -"/index.md".length); + else if (!path.endsWith(".md")) dir = path; // a bare directory (e.g. `cat .../docs/src/graph`) + + if (dir !== null) { + const meta: DocMeta[] = (await listDocMeta(query, tableName, { dirPrefix: dir })).map((r) => ({ + doc_id: r.doc_id, + version: r.version, + updated_at: r.updated_at, + status: r.status, + tier: r.tier, + })); + const directFiles = meta + .filter((m) => m.status === "active" && dirOf(m.doc_id) === dir) + .map((m) => m.doc_id); + const summaries = new Map(); + if (directFiles.length > 0) { + for (const d of await listDocsByIds(query, tableName, directFiles)) { + summaries.set(d.doc_id, firstDocLine(d.content)); + } + } + return { kind: "ok", body: buildDocsIndex(meta, dir, summaries) }; + } + + // Leaf: ".md" → doc for that file. + const docId = path.slice(0, -".md".length); + const row = await getDocLatest(query, tableName, docId); + if (!row) return { kind: "not-found", message: `${subpath}: No such file or directory` }; + const header = + `# ${row.doc_id}\n` + + `version: ${row.version} tier: ${row.tier} status: ${row.status} updated: ${row.updated_at}\n` + + `---\n`; + return { kind: "ok", body: header + row.content }; +} diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index ac2ca891..24a78a7c 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -13,6 +13,7 @@ import { log as _log } from "../utils/debug.js"; import { isDirectRun } from "../utils/direct-run.js"; import { type GrepParams, parseBashGrep, handleGrepDirect } from "./grep-direct.js"; import { handleGraphVfs } from "../graph/vfs-handler.js"; +import { handleDocsVfs } from "../docs/vfs-handler.js"; import { executeCompiledBashCommand } from "./bash-command-compiler.js"; import { findVirtualPaths, @@ -246,6 +247,7 @@ interface ClaudePreToolDeps { executeCompiledBashCommandFn?: typeof executeCompiledBashCommand; handleGrepDirectFn?: typeof handleGrepDirect; handleGraphVfsFn?: typeof handleGraphVfs; + handleDocsVfsFn?: typeof handleDocsVfs; readVirtualPathContentsFn?: typeof readVirtualPathContents; readVirtualPathContentFn?: typeof readVirtualPathContent; listVirtualPathRowsFn?: typeof listVirtualPathRows; @@ -269,6 +271,7 @@ export async function processPreToolUse(input: PreToolUseInput, deps: ClaudePreT executeCompiledBashCommandFn = executeCompiledBashCommand, handleGrepDirectFn = handleGrepDirect, handleGraphVfsFn = handleGraphVfs, + handleDocsVfsFn = handleDocsVfs, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, @@ -446,6 +449,31 @@ export async function processPreToolUse(input: PreToolUseInput, deps: ClaudePreT return buildAllowDecision(safeEchoCommand(body), `[hivemind graph] ls /graph`); } + // Docs VFS dispatch — the browsable per-directory docs index under + // /docs/. Separate surface from the memory index (no retrieval + // pollution). Backed by the docs TABLE, so this is async via api.query. + // `/docs` (bare) and `/docs/index.md` both render the root index. + if (virtualPath && (virtualPath === "/docs" || virtualPath.startsWith("/docs/")) && !virtualPath.endsWith("/")) { + const subpath = virtualPath === "/docs" ? "" : virtualPath.slice("/docs/".length); + logFn(`docs vfs: ${subpath || "(root)"}`); + const result = await handleDocsVfsFn(subpath, (sql) => api.query(sql), config.docsTableName); + const body = result.kind === "ok" ? result.body : `(${result.kind}) ${result.message}`; + if (input.tool_name === "Read") { + const file_path = writeReadCacheFileFn(input.session_id, virtualPath, body); + return buildReadDecision(file_path, `[hivemind docs] ${virtualPath}`); + } + return buildAllowDecision(safeEchoCommand(body), `[hivemind docs] /docs/${subpath}`); + } + if (lsDir === "/docs" || lsDir === "/docs/") { + const result = await handleDocsVfsFn("", (sql) => api.query(sql), config.docsTableName); + const body = result.kind === "ok" ? result.body : `(${result.kind}) ${result.message}`; + if (input.tool_name === "Read") { + const file_path = writeReadCacheFileFn(input.session_id, "/docs/_listing.txt", body); + return buildReadDecision(file_path, "[hivemind docs] ls /docs"); + } + return buildAllowDecision(safeEchoCommand(body), `[hivemind docs] ls /docs`); + } + if (virtualPath && !virtualPath.endsWith("/")) { logFn(`direct read: ${virtualPath}`); let content = virtualPath === "/index.md" diff --git a/tests/claude-code/pre-tool-use-branches.test.ts b/tests/claude-code/pre-tool-use-branches.test.ts index a4f0588a..9b7d0ba7 100644 --- a/tests/claude-code/pre-tool-use-branches.test.ts +++ b/tests/claude-code/pre-tool-use-branches.test.ts @@ -205,6 +205,41 @@ describe("extractGrepParams", () => { }); }); +describe("processPreToolUse: docs VFS routing", () => { + const DOCS_CONFIG = { ...BASE_CONFIG, docsTableName: "hivemind_docs" } as any; + + it("routes a Read of /docs/index.md to the docs VFS handler (root subpath)", async () => { + const handleDocsVfsFn = vi.fn(async () => ({ kind: "ok" as const, body: "# Docs Index\n" })); + const d = await processPreToolUse( + { session_id: "s", tool_name: "Read", tool_input: { file_path: `${MEM_ABS}/docs/index.md` }, tool_use_id: "t" }, + { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn }, + ); + expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs"); + // Read must get a file_path-shaped decision (cache file), not a real command. + expect(d?.file_path).toContain("docs/index.md"); + expect(d?.command).toBe(""); + }); + + it("extracts the nested subpath for a drill-down read", async () => { + const handleDocsVfsFn = vi.fn(async () => ({ kind: "ok" as const, body: "x" })); + await processPreToolUse( + { session_id: "s", tool_name: "Read", tool_input: { file_path: `${MEM_ABS}/docs/src/graph/diff.ts.md` }, tool_use_id: "t" }, + { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn }, + ); + expect(handleDocsVfsFn).toHaveBeenCalledWith("src/graph/diff.ts.md", expect.any(Function), "hivemind_docs"); + }); + + it("routes a Bash cat of a docs path to a command-shaped allow decision", async () => { + const handleDocsVfsFn = vi.fn(async () => ({ kind: "ok" as const, body: "# Docs Index\n" })); + const d = await processPreToolUse( + { session_id: "s", tool_name: "Bash", tool_input: { command: "cat ~/.deeplake/memory/docs/index.md" }, tool_use_id: "t" }, + { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn, logFn: vi.fn() }, + ); + expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs"); + expect(d?.command).toContain("Docs Index"); + }); +}); + describe("processPreToolUse: non-memory / no-op paths", () => { it("returns null when the command doesn't touch memory and there's no shellCmd", async () => { const d = await processPreToolUse( diff --git a/tests/shared/docs-vfs-handler.test.ts b/tests/shared/docs-vfs-handler.test.ts new file mode 100644 index 00000000..84e5b5f1 --- /dev/null +++ b/tests/shared/docs-vfs-handler.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from "vitest"; + +// Pass-through the read-stability gate so SQL shapes stay exact (see docs.test.ts). +vi.mock("../../src/docs/stable-read.js", () => ({ + stableUnionRows: (q: (sql: string) => unknown, sql: string) => q(sql), +})); + +import { handleDocsVfs } from "../../src/docs/vfs-handler.js"; + +const TBL = "hivemind_docs"; + +function metaRow(doc_id: string, over: Record = {}) { + return { id: `id-${doc_id}`, doc_id, version: 1, updated_at: "2026-07-01T10:00:00.000Z", status: "active", tier: "fast", ...over }; +} +function fullRow(doc_id: string, content: string, over: Record = {}) { + return { + id: `id-${doc_id}`, doc_id, path: `/docs/p/${doc_id}.md`, content, + anchors: "[]", tier: "fast", status: "active", project: "p", version: 1, + created_at: "2026-07-01T10:00:00.000Z", updated_at: "2026-07-01T10:00:00.000Z", + agent: "m", plugin_version: "0", ...over, + }; +} + +/** Route the query by SQL shape so one fake stands in for all three reads. */ +function router(meta: Record[], byIds: Record[], latest: Record[]) { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + if (/SELECT id, doc_id, version/.test(sql)) return meta; + if (/doc_id IN \(/.test(sql)) return byIds; + if (/WHERE doc_id = /.test(sql)) return latest; + return []; + }); + return { calls, query }; +} + +describe("handleDocsVfs", () => { + const corpus = [metaRow("src/graph/diff.ts"), metaRow("src/graph/cache.ts"), metaRow("src/docs/read.ts")]; + + it("renders the root index (top directories) for '' and 'index.md'", async () => { + for (const sub of ["", "index.md"]) { + const { query } = router(corpus, [], []); + const r = await handleDocsVfs(sub, query, TBL); + expect(r.kind).toBe("ok"); + expect(r.kind === "ok" && r.body).toContain("# Docs Index"); + expect(r.kind === "ok" && r.body).toContain("[src/](src/index.md)"); + } + }); + + it("renders a directory index scoped by prefix, with summaries for direct files", async () => { + const dirMeta = [metaRow("src/graph/diff.ts"), metaRow("src/graph/extract/python.ts")]; + const { calls, query } = router(dirMeta, [fullRow("src/graph/diff.ts", "# diff\n\nCompares snapshots.")], []); + const r = await handleDocsVfs("src/graph/index.md", query, TBL); + expect(r.kind).toBe("ok"); + // Metadata read is directory-scoped. + expect(calls.some((c) => c.includes("doc_id LIKE 'src/graph/%'"))).toBe(true); + if (r.kind === "ok") { + expect(r.body).toContain("# Docs: src/graph/"); + expect(r.body).toContain("[extract/](extract/index.md)"); // subdir + expect(r.body).toContain("Compares snapshots."); // summary of the direct file + } + }); + + it("returns the doc content for a leaf '.md'", async () => { + const { calls, query } = router([], [], [fullRow("src/graph/diff.ts", "# diff\n\nThe diff doc body.")]); + const r = await handleDocsVfs("src/graph/diff.ts.md", query, TBL); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") { + expect(r.body).toContain("# src/graph/diff.ts"); + expect(r.body).toContain("The diff doc body."); + } + // It resolved by exact doc_id, stripping the .md suffix. + expect(calls.some((c) => c.includes("WHERE doc_id = 'src/graph/diff.ts'"))).toBe(true); + }); + + it("returns not-found for a leaf whose doc does not exist", async () => { + const { query } = router([], [], []); + const r = await handleDocsVfs("src/graph/nope.ts.md", query, TBL); + expect(r.kind).toBe("not-found"); + expect(r.kind === "not-found" && r.message).toContain("No such file"); + }); +}); From f7f5c82ca7a1760bc073a2f19df8eaa0e24dd649 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 15:14:35 +0000 Subject: [PATCH 25/39] refactor: update docs in place instead of insert-only append --- src/docs/write.ts | 84 +++++++++++++++++-------------- tests/shared/docs-refresh.test.ts | 12 ++--- tests/shared/docs.test.ts | 37 +++++++------- 3 files changed, 71 insertions(+), 62 deletions(-) diff --git a/src/docs/write.ts b/src/docs/write.ts index 67316f1e..6d1ebe25 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -1,21 +1,22 @@ /** - * Write helpers for `hivemind_docs` — INSERT-only against the immutable - * skills/rules-table pattern. Every edit appends a fresh row with version+1; - * we never UPDATE. Reads (see ./read.ts) pick the latest version per doc_id. + * Write helpers for `hivemind_docs` — ONE row per doc, mutated in place. * - * Why no UPDATEs: the Deeplake backend silently coalesces two rapid UPDATEs - * on the same row (see CLAUDE.md "UPDATE coalescing quirk"). INSERT-only - * sidesteps the bug entirely. See `src/rules/write.ts` for the precedent and - * `deeplake-schema.ts` DOCS_COLUMNS for the column list. + * A brand-new doc is INSERTed at version=1 (`insertDoc`); every later edit is a + * single UPDATE of that same row (`updateInPlace`), bumping `version` as an + * in-row counter. Reads (see ./read.ts) resolve one row per `doc_id`. * - * Differences from rules: - * - `doc_id` is the documented source file path, supplied by the caller - * (the file path IS the stable identity), not a generated UUID. - * - `content` is markdown and MAY contain newlines — they are preserved, - * not rejected (unlike rule bodies, which are single-line). - * - `created_at` is an immutable creation timestamp carried across every - * version bump; only `updated_at` advances. This mirrors the - * timestamp-preservation pattern used by goals/skills. + * History: this used to be INSERT-only version-append, to dodge a Deeplake + * backend bug that coalesced two rapid UPDATEs on the same row. F0 verified + * (repro: sequential rapid single-row UPDATEs, 0 losses) that a SINGLE UPDATE + * setting all columns at once is safe — the bug only bit *separate* rapid + * UPDATEs. So we moved to UPDATE-in-place: no unbounded version growth, and a + * trivial one-row read path. The invariant `updateInPlace` upholds: exactly one + * UPDATE per write, all columns together. + * + * - `doc_id` is the documented source file path (the stable identity), and is + * never mutated. + * - `content` is markdown and MAY contain newlines — preserved, not rejected. + * - `created_at` is immutable; only `updated_at` advances. */ import { randomUUID } from "node:crypto"; @@ -219,7 +220,7 @@ export async function editDoc( if (!previous) { throw new Error(`Doc not found: ${input.doc_id}`); } - return appendVersion(query, tableName, previous, input); + return updateInPlace(query, tableName, previous, input); } /** @@ -250,7 +251,7 @@ export async function setDoc( plugin_version: input.plugin_version, }); } - return appendVersion(query, tableName, previous, { + return updateInPlace(query, tableName, previous, { doc_id: input.doc_id, content: input.content, anchors: input.anchors, @@ -282,7 +283,20 @@ export async function archiveDoc( }); } -async function appendVersion( +/** + * Update a doc IN PLACE — one row per `doc_id`, mutated with a single UPDATE. + * + * This replaced the old INSERT-only version-append once the Deeplake backend's + * UPDATE-coalescing bug was verified fixed for our access pattern (F0): a + * single UPDATE that sets ALL columns at once, applied sequentially per doc, is + * safe (0 losses over the repro). The historic bug only bit *two separate* + * rapid UPDATEs to the same row — which we never do here. + * + * `version` is bumped as an in-row update counter; `created_at` is immutable; + * `updated_at` advances. The row is targeted by its unique `id` so a table that + * still carries pre-migration history rows updates exactly the current one. + */ +async function updateInPlace( query: QueryFn, tableName: string, previous: DocRow, @@ -291,7 +305,6 @@ async function appendVersion( const content = next.content ?? previous.content; assertValidContent(content); const safe = sqlIdent(tableName); - const rowId = randomUUID(); const now = new Date().toISOString(); const nextVersion = previous.version + 1; const anchors = serializeAnchors(next.anchors ?? previous.anchors); @@ -300,26 +313,21 @@ async function appendVersion( const path = next.path ?? previous.path; const project = next.project ?? previous.project; + // One UPDATE, all columns — the F0 safety rule. created_at + doc_id are not + // touched (immutable identity/creation stamp). const sql = - `INSERT INTO "${safe}" ` + - `(id, doc_id, path, content, anchors, tier, status, project, version, ` + - `created_at, updated_at, agent, plugin_version) ` + - `VALUES (` + - `'${sqlStr(rowId)}', ` + - `'${sqlStr(previous.doc_id)}', ` + - `'${sqlStr(path)}', ` + - `E'${sqlStr(content)}', ` + - `E'${sqlStr(anchors)}', ` + - `'${sqlStr(tier)}', ` + - `'${sqlStr(status)}', ` + - `'${sqlStr(project)}', ` + - `${nextVersion}, ` + - // created_at carried from the original row — immutable creation stamp. - `'${sqlStr(previous.created_at)}', ` + - `'${sqlStr(now)}', ` + - `'${sqlStr(next.agent ?? "manual")}', ` + - `'${sqlStr(next.plugin_version ?? "")}'` + - `)`; + `UPDATE "${safe}" SET ` + + `path = '${sqlStr(path)}', ` + + `content = E'${sqlStr(content)}', ` + + `anchors = E'${sqlStr(anchors)}', ` + + `tier = '${sqlStr(tier)}', ` + + `status = '${sqlStr(status)}', ` + + `project = '${sqlStr(project)}', ` + + `version = ${nextVersion}, ` + + `updated_at = '${sqlStr(now)}', ` + + `agent = '${sqlStr(next.agent ?? "manual")}', ` + + `plugin_version = '${sqlStr(next.plugin_version ?? "")}' ` + + `WHERE id = '${sqlStr(previous.id)}'`; await query(sql); return { doc_id: previous.doc_id, version: nextVersion }; } diff --git a/tests/shared/docs-refresh.test.ts b/tests/shared/docs-refresh.test.ts index df01a2f4..b14ea90b 100644 --- a/tests/shared/docs-refresh.test.ts +++ b/tests/shared/docs-refresh.test.ts @@ -163,10 +163,10 @@ describe("refreshDocs", () => { expect(report.refreshed).toBe(1); expect(report.outcomes[0]).toMatchObject({ doc_id: "a.ts", status: "refreshed", version: 4 }); expect(generate).toHaveBeenCalledOnce(); - // 2 queries: getDocLatest + INSERT. The INSERT carries the FRESH anchor - // (recomputed from current code), not the stale stored hash. + // 2 queries: getDocLatest + UPDATE-in-place. The UPDATE carries the FRESH + // anchor (recomputed from current code), not the stale stored hash. expect(calls).toHaveLength(2); - expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + expect(calls[1]).toMatch(/^UPDATE "hivemind_docs" SET/); expect(calls[1]).toContain("new doc body"); expect(calls[1]).toContain(buildAnchor(foo, dir)!.content_hash); expect(calls[1]).not.toContain("stale"); @@ -245,10 +245,10 @@ describe("refreshDocs", () => { expect(report.outcomes[0]).toMatchObject({ doc_id: "a.ts", status: "archived", version: 4 }); // No token spent on a deleted file. expect(generate).not.toHaveBeenCalled(); - // archiveDoc = getDocLatest + INSERT(status='archived'); nothing else. + // archiveDoc = getDocLatest + UPDATE(status='archived'); nothing else. expect(calls).toHaveLength(2); - expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); - expect(calls[1]).toContain("'archived'"); + expect(calls[1]).toMatch(/^UPDATE "hivemind_docs" SET/); + expect(calls[1]).toContain("status = 'archived'"); }); it("drops a dangling anchor when its symbol vanished from the graph", async () => { diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 29e36ca3..3b067fba 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -240,24 +240,24 @@ describe("insertDocResilient", () => { // ── editDoc ─────────────────────────────────────────────────────────────────── describe("editDoc", () => { - it("reads latest, then INSERTs version+1 carrying the IMMUTABLE created_at while updated_at advances", async () => { + it("reads latest, then UPDATEs in place bumping version; created_at untouched, updated_at advances", async () => { const { calls, query } = mockQuery([ - () => [fakeRow({ version: 1, content: "old", created_at: "2026-05-20T10:00:00.000Z" })], + () => [fakeRow({ id: "row-1", version: 1, content: "old", created_at: "2026-05-20T10:00:00.000Z" })], () => [], ]); const result = await editDoc(query, TBL, { doc_id: "src/shell/deeplake-fs.ts", content: "new" }); expect(result).toEqual({ doc_id: "src/shell/deeplake-fs.ts", version: 2 }); expect(calls).toHaveLength(2); expect(calls[0]).toMatch(/^SELECT .* FROM "hivemind_docs" WHERE doc_id = 'src\/shell\/deeplake-fs.ts'$/); - expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + // UPDATE-in-place, targeting the exact row by id. + expect(calls[1]).toMatch(/^UPDATE "hivemind_docs" SET/); expect(calls[1]).toContain(`E'new'`); - expect(calls[1]).toContain(", 2, "); - // The original creation stamp is carried forward verbatim... - expect(calls[1]).toContain("'2026-05-20T10:00:00.000Z'"); - // ...and updated_at is a DIFFERENT, newer timestamp (now). - const stamps = calls[1].match(/'(\d{4}-\d{2}-\d{2}T[\d:.]+Z)'/g)!; - expect(stamps[0]).toBe("'2026-05-20T10:00:00.000Z'"); // created_at - expect(stamps[1]).not.toBe("'2026-05-20T10:00:00.000Z'"); // updated_at = now + expect(calls[1]).toContain("version = 2"); + expect(calls[1]).toContain(`WHERE id = 'row-1'`); + // created_at is immutable → the UPDATE must NOT touch it. + expect(calls[1]).not.toContain("created_at"); + // updated_at advances to a fresh "now" timestamp. + expect(calls[1]).toMatch(/updated_at = '\d{4}-\d{2}-\d{2}T[\d:.]+Z'/); }); it("carries over previous content + anchors when only status changes", async () => { @@ -343,9 +343,9 @@ describe("setDoc", () => { expect(calls[1]).not.toContain("'old-proj'"); }); - it("APPENDS version+1 when the doc_id already exists — never forks a second v1", async () => { + it("UPDATEs the existing row in place (bumping version), never a second row", async () => { const { calls, query } = mockQuery([ - () => [fakeRow({ doc_id: "src/a.ts", version: 4, created_at: "2026-01-01T00:00:00.000Z" })], + () => [fakeRow({ id: "row-9", doc_id: "src/a.ts", version: 4, created_at: "2026-01-01T00:00:00.000Z" })], () => [], ]); const result = await setDoc(query, TBL, { @@ -356,12 +356,13 @@ describe("setDoc", () => { }); expect(result).toEqual({ doc_id: "src/a.ts", version: 5 }); expect(calls).toHaveLength(2); - // The INSERT is version 5, NOT a second version 1 — the fork-history bug - // Codex flagged is impossible through setDoc. - expect(calls[1]).toContain(", 5, "); - expect(calls[1]).not.toMatch(/, 1, /); - // immutable created_at carried from the existing chain - expect(calls[1]).toContain("'2026-01-01T00:00:00.000Z'"); + // A single UPDATE of the existing row — one row per doc, no new INSERT. + expect(calls[1]).toMatch(/^UPDATE "hivemind_docs" SET/); + expect(calls[1]).not.toMatch(/^INSERT/); + expect(calls[1]).toContain("version = 5"); + expect(calls[1]).toContain(`WHERE id = 'row-9'`); + // created_at is immutable → not part of the UPDATE. + expect(calls[1]).not.toContain("created_at"); expect(calls[1]).toContain(`E'updated'`); }); }); From 041ce5219007e4c87ba8df66db0b6b21e5a38869 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 15:20:17 +0000 Subject: [PATCH 26/39] perf: scope docs refresh to the git diff, not the whole corpus --- src/commands/docs.ts | 17 ++++++- src/docs/candidates.ts | 76 ++++++++++++++++++++++++++++ src/docs/index.ts | 3 ++ tests/shared/docs-candidates.test.ts | 51 +++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/docs/candidates.ts create mode 100644 tests/shared/docs-candidates.test.ts diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 64b193e9..45d47fe4 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -40,6 +40,8 @@ import { buildDocsIndex, dirOf, firstDocLine, + changedFilesFromGit, + expandToCandidateFiles, type DocRow, type DocMeta, type DocTier, @@ -372,9 +374,22 @@ export async function runDocsCommand(args: string[]): Promise { process.exit(1); throw new Error("unreachable"); } + // Scope the read to the diff when git can tell us what changed: load only + // the candidate docs (changed files + their transitive callers) instead of + // the whole corpus. Per-commit work becomes O(diff), not O(all docs). No + // git signal → full scan, logged (never a silent narrowing). + const changed = changedFilesFromGit(cwd); let docs: DocRow[] = []; try { - docs = await listDocs(query, tableName, { status: "active", limit: 100000 }); + if (changed !== null) { + const candidates = expandToCandidateFiles(snap, changed); + docs = await listDocsByIds(query, tableName, candidates); + docs = docs.filter((d) => d.status === "active"); + console.error(`[docs refresh] scoped to ${candidates.length} candidate file(s) from git diff (${changed.length} changed)`); + } else { + docs = await listDocs(query, tableName, { status: "active", limit: 100000 }); + console.error(`[docs refresh] no git signal — full scan of ${docs.length} doc(s)`); + } } catch (err) { if (!isMissingTableError((err as Error).message)) throw err; } diff --git a/src/docs/candidates.ts b/src/docs/candidates.ts new file mode 100644 index 00000000..4af631fd --- /dev/null +++ b/src/docs/candidates.ts @@ -0,0 +1,76 @@ +/** + * Scope a refresh to the diff — turn "what changed in git" into "which docs + * could possibly be stale", so per-commit work is O(diff), not O(all docs). + * + * Two steps: + * 1. `changedFilesFromGit` — the files touched vs HEAD. Union of the working + * tree (uncommitted edits — the manual `docs refresh` case) and the last + * commit (HEAD~1..HEAD — the post-commit auto-refresh case). Returns null + * when git is unavailable, so the caller can fall back to a full scan. + * 2. `expandToCandidateFiles` — the changed files PLUS every file that + * transitively CALLS a symbol in them (blast radius over the graph's + * reverse edges). Loading exactly these docs reproduces the full-scan + * impacted set: the direct hash pass flags the changed files' docs, and + * their caller docs are present to be flagged by the relational pass. + */ + +import { execFileSync } from "node:child_process"; +import { impactedNodes } from "../graph/render/impact.js"; +import type { GraphSnapshot } from "../graph/types.js"; + +/** Run `git ` in `cwd`; returns stdout, or null on any failure. */ +export type GitRunner = (args: string[]) => string | null; + +function defaultGit(cwd: string): GitRunner { + return (args) => { + try { + return execFileSync("git", ["-C", cwd, ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + } catch { + return null; + } + }; +} + +function collect(out: string | null, into: Set): void { + if (out === null) return; + for (const line of out.split("\n")) { + const f = line.trim(); + if (f) into.add(f); + } +} + +/** + * Files changed relative to HEAD. `null` means "no git signal" → full scan. + * An empty array means git works but nothing changed. + */ +export function changedFilesFromGit(cwd: string, git: GitRunner = defaultGit(cwd)): string[] | null { + const workingTree = git(["diff", "--name-only", "HEAD"]); + if (workingTree === null) return null; // not a repo / git missing + const files = new Set(); + collect(workingTree, files); + // Also the last commit, for the post-commit path where the tree is clean. + collect(git(["diff", "--name-only", "HEAD~1", "HEAD"]), files); + return [...files]; +} + +/** + * Expand changed files to the candidate doc set: the changed files + the + * transitive callers of any symbol they define. Result is a superset of the + * files whose docs the full scan would flag — never smaller. + */ +export function expandToCandidateFiles(snap: GraphSnapshot, changedFiles: Iterable): string[] { + const changed = new Set(changedFiles); + const out = new Set(changed); + const seedIds = snap.nodes.filter((n) => changed.has(n.source_file)).map((n) => n.id); + if (seedIds.length > 0) { + const byId = new Map(snap.nodes.map((n) => [n.id, n])); + for (const id of impactedNodes(snap, seedIds)) { + const node = byId.get(id); + if (node) out.add(node.source_file); + } + } + return [...out]; +} diff --git a/src/docs/index.ts b/src/docs/index.ts index c0133d61..31d6f4c7 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -22,6 +22,9 @@ export type { export { buildDocsIndex, dirOf, firstDocLine } from "./index-render.js"; export type { DocMeta } from "./index-render.js"; +export { changedFilesFromGit, expandToCandidateFiles } from "./candidates.js"; +export type { GitRunner } from "./candidates.js"; + export { parseSourceLocation, readSymbolSource, diff --git a/tests/shared/docs-candidates.test.ts b/tests/shared/docs-candidates.test.ts new file mode 100644 index 00000000..0a686ed9 --- /dev/null +++ b/tests/shared/docs-candidates.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from "vitest"; +import { changedFilesFromGit, expandToCandidateFiles } from "../../src/docs/candidates.js"; +import type { GraphNode, GraphSnapshot } from "../../src/graph/types.js"; + +function node(id: string, source_file: string): GraphNode { + return { id, label: id, kind: "function", source_file, source_location: "L1", language: "typescript", exported: true }; +} +function snap(nodes: GraphNode[], links: Array<{ source: string; target: string; relation: string }> = []): GraphSnapshot { + return { nodes, links } as unknown as GraphSnapshot; +} + +describe("changedFilesFromGit", () => { + it("unions working-tree changes with the last commit, deduped", async () => { + const git = vi.fn((args: string[]) => { + if (args.join(" ") === "diff --name-only HEAD") return "src/money.ts\nsrc/cart.ts\n"; + if (args.join(" ") === "diff --name-only HEAD~1 HEAD") return "src/cart.ts\nsrc/util.ts\n"; + return ""; + }); + const out = changedFilesFromGit("/x", git)!; + expect(new Set(out)).toEqual(new Set(["src/money.ts", "src/cart.ts", "src/util.ts"])); + }); + + it("returns null when git is unavailable (→ caller does a full scan)", () => { + const git = vi.fn(() => null); // not a repo / git missing + expect(changedFilesFromGit("/x", git)).toBeNull(); + }); + + it("returns [] when git works but nothing changed", () => { + const git = vi.fn(() => ""); + expect(changedFilesFromGit("/x", git)).toEqual([]); + }); +}); + +describe("expandToCandidateFiles", () => { + // cart.ts:total calls money.ts:addTax → editing money.ts must pull cart.ts in. + const s = snap( + [node("src/money.ts:addTax:function", "src/money.ts"), node("src/cart.ts:total:function", "src/cart.ts"), node("src/other.ts:x:function", "src/other.ts")], + [{ source: "src/cart.ts:total:function", target: "src/money.ts:addTax:function", relation: "calls" }], + ); + + it("includes the changed file AND its transitive callers", () => { + const out = new Set(expandToCandidateFiles(s, ["src/money.ts"])); + expect(out.has("src/money.ts")).toBe(true); // the changed file + expect(out.has("src/cart.ts")).toBe(true); // caller of addTax + expect(out.has("src/other.ts")).toBe(false); // unrelated → not loaded + }); + + it("returns just the changed files when they define no graph symbols", () => { + expect(expandToCandidateFiles(s, ["README.md"])).toEqual(["README.md"]); + }); +}); From f7108ffdec80933ab8137b3738e124e425a59244 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 15:26:15 +0000 Subject: [PATCH 27/39] feat: make the docs LLM invocation pluggable per host agent --- src/commands/docs.ts | 6 +-- src/docs/refresh-llm.ts | 72 +++++++++++++++++++++++++-- tests/shared/docs-refresh-llm.test.ts | 36 +++++++++++++- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 45d47fe4..0ccf30da 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -47,7 +47,7 @@ import { type DocTier, type DocAnchor, } from "../docs/index.js"; -import { makeClaudeGenerate, makeClaudeGenerateDoc } from "../docs/refresh-llm.js"; +import { makeHostGenerate, makeHostGenerateDoc } from "../docs/refresh-llm.js"; import { generateDocs, selectTargets, type GenScope } from "../docs/generate.js"; import { loadCurrentSnapshot } from "../graph/load-current.js"; import { isMissingTableError } from "../deeplake-schema.js"; @@ -415,7 +415,7 @@ export async function runDocsCommand(args: string[]): Promise { repoRoot: cwd, impacted, docsById, - generate: makeClaudeGenerate(), + generate: makeHostGenerate(), agent: cfg.userName, pluginVersion, }); @@ -476,7 +476,7 @@ export async function runDocsCommand(args: string[]): Promise { const report = await generateDocs({ query, tableName, snap, repoRoot: cwd, project, scope, include, exclude, existing, force, limit, concurrency, - generate: makeClaudeGenerateDoc(), agent: cfg.userName, pluginVersion, + generate: makeHostGenerateDoc(), agent: cfg.userName, pluginVersion, }); console.log(`Generated ${report.created}, skipped ${report.skipped}, failed ${report.failed} (of ${report.targets} targets).`); for (const o of report.outcomes) { diff --git a/src/docs/refresh-llm.ts b/src/docs/refresh-llm.ts index b0a4fb81..7e200e90 100644 --- a/src/docs/refresh-llm.ts +++ b/src/docs/refresh-llm.ts @@ -13,7 +13,7 @@ */ import { execFileSync } from "node:child_process"; -import { buildClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; +import { buildClaudeInvocation, buildTrailingPromptInvocation, type ClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; import { resolveCliBin } from "../utils/resolve-cli-bin.js"; import { buildRefreshPrompt, type GenerateFn } from "./refresh.js"; import { buildGeneratePrompt, type GenerateDocFn } from "./generate.js"; @@ -31,9 +31,54 @@ export function unwrapModelOutput(raw: string): string { return fence ? fence[1].trim() : text; } -/** Run a single prompt through the host `claude` CLI and return the unwrapped output. */ -export function runClaudePrompt(bin: string, prompt: string, timeoutMs = 120_000): string { - const inv = buildClaudeInvocation(bin, prompt); +/** + * Which host CLI rewrites/authors docs. The auto-refresh runs from a host + * agent's post-commit hook, so `claude -p` is wrong on a codex/cursor/… box. + * A spec names the CLI to resolve on PATH and how to shape its invocation. + */ +export interface DocLlmSpec { + label: string; + bin: string; + build: (bin: string, prompt: string) => ClaudeInvocation; +} + +const REGISTRY: Record = { + claude: { label: "claude", bin: "claude", build: (b, p) => buildClaudeInvocation(b, p) }, + codex: { + label: "codex", + bin: "codex", + build: (b, p) => buildTrailingPromptInvocation(b, ["exec", "--dangerously-bypass-approvals-and-sandbox"], p), + }, +}; + +/** + * Resolve the doc-LLM spec from the environment: + * - `HIVEMIND_DOCS_LLM_BIN` (+ optional `HIVEMIND_DOCS_LLM_FLAGS`, comma-sep) + * → a fully custom CLI (prompt appended as the trailing arg). Escape hatch + * for any agent not in the registry. + * - `HIVEMIND_DOCS_LLM_AGENT` = claude | codex → a named registry entry. + * - default → claude (byte-identical to the previous behavior). + */ +export function resolveDocLlmSpec(env: NodeJS.ProcessEnv = process.env): DocLlmSpec { + const customBin = env.HIVEMIND_DOCS_LLM_BIN; + if (customBin && customBin.trim() !== "") { + const flags = (env.HIVEMIND_DOCS_LLM_FLAGS ?? "").split(",").map((s) => s.trim()).filter(Boolean); + return { label: `custom:${customBin}`, bin: customBin, build: (b, p) => buildTrailingPromptInvocation(b, flags, p) }; + } + const agent = (env.HIVEMIND_DOCS_LLM_AGENT ?? "claude").toLowerCase(); + const spec = REGISTRY[agent]; + if (!spec) { + throw new Error( + `Unknown HIVEMIND_DOCS_LLM_AGENT="${agent}". Known: ${Object.keys(REGISTRY).join(", ")}. ` + + `For any other CLI set HIVEMIND_DOCS_LLM_BIN (and HIVEMIND_DOCS_LLM_FLAGS).`, + ); + } + return spec; +} + +/** Run a prompt through a resolved host-LLM spec, returning unwrapped output. */ +export function runHostPrompt(spec: DocLlmSpec, bin: string, prompt: string, timeoutMs = 120_000): string { + const inv = spec.build(bin, prompt); const out = execFileSync(inv.file, inv.args, { ...inv.options, encoding: "utf-8", @@ -43,6 +88,25 @@ export function runClaudePrompt(bin: string, prompt: string, timeoutMs = 120_000 return unwrapModelOutput((out ?? "").toString()); } +/** Doc REFRESH generator backed by the resolved host agent (claude/codex/custom). */ +export function makeHostGenerate(timeoutMs = 120_000, env: NodeJS.ProcessEnv = process.env): GenerateFn { + const spec = resolveDocLlmSpec(env); + const bin = resolveCliBin(spec.bin); + return async (ctx) => runHostPrompt(spec, bin, buildRefreshPrompt(ctx), timeoutMs); +} + +/** Fresh doc GENERATION generator backed by the resolved host agent. */ +export function makeHostGenerateDoc(timeoutMs = 120_000, env: NodeJS.ProcessEnv = process.env): GenerateDocFn { + const spec = resolveDocLlmSpec(env); + const bin = resolveCliBin(spec.bin); + return async (input) => runHostPrompt(spec, bin, buildGeneratePrompt(input), timeoutMs); +} + +/** Run a single prompt through the host `claude` CLI and return the unwrapped output. */ +export function runClaudePrompt(bin: string, prompt: string, timeoutMs = 120_000): string { + return runHostPrompt(REGISTRY.claude, bin, prompt, timeoutMs); +} + /** Resolve the claude binary once (PATH lookup), with the usual fallback. */ export function resolveClaudeBin(claudeBin?: string): string { return claudeBin ?? resolveCliBin("claude"); diff --git a/tests/shared/docs-refresh-llm.test.ts b/tests/shared/docs-refresh-llm.test.ts index 7c78d464..ad4c4fa9 100644 --- a/tests/shared/docs-refresh-llm.test.ts +++ b/tests/shared/docs-refresh-llm.test.ts @@ -6,7 +6,7 @@ vi.mock("node:child_process", () => ({ execFileSync: (...args: unknown[]) => execFileSyncMock(...args), })); -import { makeClaudeGenerate, unwrapModelOutput } from "../../src/docs/refresh-llm.js"; +import { makeClaudeGenerate, resolveDocLlmSpec, unwrapModelOutput } from "../../src/docs/refresh-llm.js"; import type { RefreshContext } from "../../src/docs/index.js"; const ctx: RefreshContext = { @@ -21,6 +21,40 @@ const ctx: RefreshContext = { beforeEach(() => execFileSyncMock.mockReset()); +describe("resolveDocLlmSpec (per-agent LLM seam)", () => { + it("defaults to claude — a `-p ` invocation (unchanged behavior)", () => { + const spec = resolveDocLlmSpec({}); + expect(spec.label).toBe("claude"); + expect(spec.bin).toBe("claude"); + const inv = spec.build("/usr/bin/claude", "PROMPT"); + expect(inv.file).toBe("/usr/bin/claude"); + expect(inv.args.slice(0, 2)).toEqual(["-p", "PROMPT"]); + }); + + it("HIVEMIND_DOCS_LLM_AGENT=codex → `codex exec … ` (prompt trailing)", () => { + const spec = resolveDocLlmSpec({ HIVEMIND_DOCS_LLM_AGENT: "codex" }); + expect(spec.bin).toBe("codex"); + const inv = spec.build("/usr/bin/codex", "PROMPT"); + expect(inv.args).toEqual(["exec", "--dangerously-bypass-approvals-and-sandbox", "PROMPT"]); + }); + + it("HIVEMIND_DOCS_LLM_BIN (+FLAGS) → a fully custom CLI, prompt as the last arg", () => { + const spec = resolveDocLlmSpec({ HIVEMIND_DOCS_LLM_BIN: "my-llm", HIVEMIND_DOCS_LLM_FLAGS: "run,--json" }); + expect(spec.bin).toBe("my-llm"); + const inv = spec.build("my-llm", "PROMPT"); + expect(inv.args).toEqual(["run", "--json", "PROMPT"]); + }); + + it("the custom escape hatch beats a named agent when both are set", () => { + const spec = resolveDocLlmSpec({ HIVEMIND_DOCS_LLM_BIN: "x", HIVEMIND_DOCS_LLM_AGENT: "codex" }); + expect(spec.label).toBe("custom:x"); + }); + + it("throws a helpful error on an unknown agent", () => { + expect(() => resolveDocLlmSpec({ HIVEMIND_DOCS_LLM_AGENT: "gpt5" })).toThrow(/Unknown HIVEMIND_DOCS_LLM_AGENT="gpt5"/); + }); +}); + describe("makeClaudeGenerate", () => { it("returns the model's stdout, trimmed", async () => { execFileSyncMock.mockReturnValue(" # Updated doc\nbody\n\n"); From 9edaed828d46f636228ef6eb24b4b58cf34c2fd1 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 15:30:55 +0000 Subject: [PATCH 28/39] feat: auto-generate docs for newly added files on refresh --- src/commands/docs.ts | 84 +++++++++++++++++++--------- src/docs/candidates.ts | 5 +- tests/shared/docs-candidates.test.ts | 7 +++ 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 0ccf30da..6691762b 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -394,38 +394,68 @@ export async function runDocsCommand(args: string[]): Promise { if (!isMissingTableError((err as Error).message)) throw err; } const impacted = computeImpactedDocs({ snap, docs, repoRoot: cwd }); - if (impacted.length === 0) { - console.log("(no docs need refreshing — all anchors fresh)"); - return; - } if (dryRun) { - console.log(`${impacted.length} doc(s) would be refreshed:`); - for (const i of impacted) { - console.log(` ${i.doc_id} [${i.reasons.map((r) => r.kind).join(", ")}]`); + if (impacted.length === 0) { + console.log("(no docs need refreshing — all anchors fresh)"); + } else { + console.log(`${impacted.length} doc(s) would be refreshed:`); + for (const i of impacted) { + console.log(` ${i.doc_id} [${i.reasons.map((r) => r.kind).join(", ")}]`); + } } return; } - // Real refresh writes — ensure the table now (dry-run above already returned). + // Real run — ensure the table now (dry-run above already returned). await api.ensureDocsTable(tableName); - const docsById = new Map(docs.map((d) => [d.doc_id, d])); - const report = await refreshDocs({ - query, - tableName, - snap, - repoRoot: cwd, - impacted, - docsById, - generate: makeHostGenerate(), - agent: cfg.userName, - pluginVersion, - }); - console.log( - `Refreshed ${report.refreshed}, archived ${report.archived}, rejected ${report.rejected}, skipped ${report.skipped}.`, - ); - for (const o of report.outcomes) { - if (o.status === "refreshed") console.log(` refreshed ${o.doc_id} → v${o.version}`); - else if (o.status === "archived") console.log(` archived ${o.doc_id} → v${o.version} (${(o.reasons ?? []).join("; ")})`); - else console.log(` ${o.status} ${o.doc_id}: ${(o.reasons ?? []).join("; ")}`); + if (impacted.length === 0) { + console.log("(no docs need refreshing — all anchors fresh)"); + } else { + const docsById = new Map(docs.map((d) => [d.doc_id, d])); + const report = await refreshDocs({ + query, + tableName, + snap, + repoRoot: cwd, + impacted, + docsById, + generate: makeHostGenerate(), + agent: cfg.userName, + pluginVersion, + }); + console.log( + `Refreshed ${report.refreshed}, archived ${report.archived}, rejected ${report.rejected}, skipped ${report.skipped}.`, + ); + for (const o of report.outcomes) { + if (o.status === "refreshed") console.log(` refreshed ${o.doc_id} → v${o.version}`); + else if (o.status === "archived") console.log(` archived ${o.doc_id} → v${o.version} (${(o.reasons ?? []).join("; ")})`); + else console.log(` ${o.status} ${o.doc_id}: ${(o.reasons ?? []).join("; ")}`); + } + } + + // Self-complete the corpus: a commit that ADDS a documentable file leaves it + // without a doc (refresh only updates existing ones). Generate docs for the + // changed files that have none — scoped to the git diff, so still O(diff). + // Modified files already have docs (refreshed above) and are skipped via + // `existing`. Only runs when git gave us a diff. + if (changed !== null && changed.length > 0) { + const existing = new Set(docs.map((d) => d.doc_id)); + const genReport = await generateDocs({ + query, + tableName, + snap, + repoRoot: cwd, + include: changed, + existing, + generate: makeHostGenerateDoc(), + agent: cfg.userName, + pluginVersion, + }); + if (genReport.created > 0) { + console.log(`Generated ${genReport.created} new doc(s) for added files:`); + for (const o of genReport.outcomes) { + if (o.status === "created") console.log(` created ${o.doc_id}`); + } + } } return; } diff --git a/src/docs/candidates.ts b/src/docs/candidates.ts index 4af631fd..cd58a107 100644 --- a/src/docs/candidates.ts +++ b/src/docs/candidates.ts @@ -51,7 +51,10 @@ export function changedFilesFromGit(cwd: string, git: GitRunner = defaultGit(cwd if (workingTree === null) return null; // not a repo / git missing const files = new Set(); collect(workingTree, files); - // Also the last commit, for the post-commit path where the tree is clean. + // Untracked, non-ignored files — a brand-new file doesn't show in `git diff` + // but is exactly the case that needs a fresh doc generated. + collect(git(["ls-files", "--others", "--exclude-standard"]), files); + // The last commit too, for the post-commit path where the tree is clean. collect(git(["diff", "--name-only", "HEAD~1", "HEAD"]), files); return [...files]; } diff --git a/tests/shared/docs-candidates.test.ts b/tests/shared/docs-candidates.test.ts index 0a686ed9..4a423dd0 100644 --- a/tests/shared/docs-candidates.test.ts +++ b/tests/shared/docs-candidates.test.ts @@ -29,6 +29,13 @@ describe("changedFilesFromGit", () => { const git = vi.fn(() => ""); expect(changedFilesFromGit("/x", git)).toEqual([]); }); + + it("includes untracked (new, non-ignored) files — the new-file case", () => { + const git = vi.fn((args: string[]) => + args.join(" ") === "ls-files --others --exclude-standard" ? "src/tax.ts\n" : "", + ); + expect(changedFilesFromGit("/x", git)).toEqual(["src/tax.ts"]); + }); }); describe("expandToCandidateFiles", () => { From efa573f9571c19f2fe21b183c6950f51b36cc438 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Wed, 1 Jul 2026 15:35:21 +0000 Subject: [PATCH 29/39] perf: refresh docs in parallel with rate-limit backoff --- src/docs/generate.ts | 13 +------ src/docs/pool.ts | 49 ++++++++++++++++++++++++ src/docs/refresh.ts | 30 ++++++++++----- tests/shared/docs-pool.test.ts | 70 ++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 src/docs/pool.ts create mode 100644 tests/shared/docs-pool.test.ts diff --git a/src/docs/generate.ts b/src/docs/generate.ts index 30d9e52b..4b57a0b8 100644 --- a/src/docs/generate.ts +++ b/src/docs/generate.ts @@ -17,6 +17,7 @@ */ import { buildAnchor, readSymbolSource } from "./anchors.js"; +import { runPool } from "./pool.js"; import { insertDocResilient, setDoc } from "./write.js"; import type { DocAnchor, QueryFn } from "./read.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -168,18 +169,6 @@ export interface GenReport { failed: number; } -/** Run `fn` over `items` with at most `n` in flight. */ -async function runPool(items: T[], n: number, fn: (item: T) => Promise): Promise { - let i = 0; - const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => { - while (i < items.length) { - const idx = i++; - await fn(items[idx]); - } - }); - await Promise.all(workers); -} - function defaultVfsPath(project: string, docId: string): string { return `/docs/${project || "default"}/${docId}.md`; } diff --git a/src/docs/pool.ts b/src/docs/pool.ts new file mode 100644 index 00000000..00a153d4 --- /dev/null +++ b/src/docs/pool.ts @@ -0,0 +1,49 @@ +/** + * Bounded concurrency helper shared by `generate` and `refresh`: run `fn` over + * `items` with at most `n` in flight. Order-independent — each worker pulls the + * next index until the list is exhausted. + */ +export async function runPool( + items: T[], + n: number, + fn: (item: T, index: number) => Promise, +): Promise { + let i = 0; + const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => { + while (i < items.length) { + const idx = i++; + await fn(items[idx], idx); + } + }); + await Promise.all(workers); +} + +/** Rate-limit / overload detection for host-LLM backoff. */ +export function isRateLimitError(err: unknown): boolean { + return err instanceof Error && /rate.?limit|429|overloaded|too many requests|quota/i.test(err.message); +} + +/** + * Run `fn`, retrying on rate-limit/overload errors with exponential backoff. + * Non-rate-limit errors surface immediately. `sleep` is injectable for tests. + */ +export async function withRateLimitRetry( + fn: () => Promise, + opts: { retries?: number; backoffMs?: number[]; sleep?: (ms: number) => Promise } = {}, +): Promise { + const retries = opts.retries ?? 3; + const backoff = opts.backoffMs ?? [1000, 4000, 10000]; + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + let lastErr: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await fn(); + } catch (err) { + if (!isRateLimitError(err)) throw err; + lastErr = err; + if (attempt === retries) break; + await sleep(backoff[Math.min(attempt, backoff.length - 1)]); + } + } + throw lastErr ?? new Error("withRateLimitRetry: exhausted retries"); +} diff --git a/src/docs/refresh.ts b/src/docs/refresh.ts index f20ec47c..f3b89733 100644 --- a/src/docs/refresh.ts +++ b/src/docs/refresh.ts @@ -18,6 +18,7 @@ import { buildAnchor, readSymbolSource } from "./anchors.js"; import { gateDocEdit, type GateResult } from "./gate.js"; +import { runPool, withRateLimitRetry } from "./pool.js"; import { archiveDoc, setDoc } from "./write.js"; import type { DocAnchor, DocRow, QueryFn } from "./read.js"; import type { ImpactedDoc, StaleReason } from "./impact.js"; @@ -70,6 +71,8 @@ export interface RefreshArgs { agent?: string; pluginVersion?: string; maxChangedLines?: number; + /** Max docs rewritten in parallel. Default 4. */ + concurrency?: number; } /** Build the prompt for one doc refresh — bounded-edit, freshness-focused. */ @@ -130,16 +133,21 @@ function gatherChangedSymbols( return out; } -/** Refresh every impacted doc, gating each edit. Pure except for `generate` + `query`. */ +/** + * Refresh every impacted doc, gating each edit. Runs a bounded worker pool so a + * commit touching many files rewrites them concurrently (each doc is an + * independent UPDATE of its own row), with rate-limit backoff around the host + * LLM. Pure except for `generate` + `query`. + */ export async function refreshDocs(args: RefreshArgs): Promise { const nodeById = new Map(args.snap.nodes.map((n) => [n.id, n])); const outcomes: RefreshOutcome[] = []; - for (const imp of args.impacted) { + await runPool(args.impacted, args.concurrency ?? 4, async (imp) => { const doc = args.docsById.get(imp.doc_id); if (!doc) { outcomes.push({ doc_id: imp.doc_id, status: "skipped", reasons: ["no current doc row"] }); - continue; + return; } // Slow-tier docs are human-curated; the gate would reject any automatic @@ -151,7 +159,7 @@ export async function refreshDocs(args: RefreshArgs): Promise { status: "rejected", reasons: ["slow-tier docs are human-curated; automatic refresh is not allowed"], }); - continue; + return; } const newAnchors = reanchor(doc, nodeById, args.repoRoot); @@ -175,17 +183,21 @@ export async function refreshDocs(args: RefreshArgs): Promise { version: res.version, reasons: ["all anchored symbols gone (file deleted/renamed)"], }); - continue; + return; } const changedSymbols = gatherChangedSymbols(imp.reasons, nodeById, args.repoRoot); let newContent: string; try { - newContent = await args.generate({ doc, reasons: imp.reasons, changedSymbols }); + // Retry the LLM call on rate-limit/overload — bulk refresh bursts hit + // the host model's limits; other errors surface immediately. + newContent = await withRateLimitRetry(() => + args.generate({ doc, reasons: imp.reasons, changedSymbols }), + ); } catch (err) { outcomes.push({ doc_id: imp.doc_id, status: "skipped", reasons: [`generate failed: ${(err as Error).message}`] }); - continue; + return; } const gate: GateResult = gateDocEdit({ @@ -198,7 +210,7 @@ export async function refreshDocs(args: RefreshArgs): Promise { }); if (!gate.ok) { outcomes.push({ doc_id: imp.doc_id, status: "rejected", reasons: gate.reasons }); - continue; + return; } const res = await setDoc(args.query, args.tableName, { @@ -212,7 +224,7 @@ export async function refreshDocs(args: RefreshArgs): Promise { plugin_version: args.pluginVersion, }); outcomes.push({ doc_id: imp.doc_id, status: "refreshed", version: res.version }); - } + }); return { outcomes, diff --git a/tests/shared/docs-pool.test.ts b/tests/shared/docs-pool.test.ts new file mode 100644 index 00000000..211bfb1c --- /dev/null +++ b/tests/shared/docs-pool.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from "vitest"; +import { runPool, isRateLimitError, withRateLimitRetry } from "../../src/docs/pool.js"; + +describe("runPool", () => { + it("processes every item and never exceeds the concurrency cap", async () => { + const items = Array.from({ length: 20 }, (_, i) => i); + const done: number[] = []; + let inFlight = 0; + let maxInFlight = 0; + await runPool(items, 4, async (n) => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((r) => setTimeout(r, 1)); + done.push(n); + inFlight--; + }); + expect(done.sort((a, b) => a - b)).toEqual(items); + expect(maxInFlight).toBeLessThanOrEqual(4); + expect(maxInFlight).toBeGreaterThan(1); // actually ran in parallel + }); + + it("passes the index and tolerates an empty list", async () => { + const seen: Array<[number, number]> = []; + await runPool(["a", "b"], 2, async (_v, i) => { seen.push([i, i]); }); + expect(seen.map(([i]) => i).sort()).toEqual([0, 1]); + await expect(runPool([], 4, async () => {})).resolves.toBeUndefined(); + }); +}); + +describe("isRateLimitError", () => { + it("matches rate-limit / 429 / overloaded / quota messages", () => { + for (const m of ["rate limit exceeded", "HTTP 429", "model overloaded", "quota exceeded", "Too Many Requests"]) { + expect(isRateLimitError(new Error(m))).toBe(true); + } + }); + it("does not match unrelated errors", () => { + expect(isRateLimitError(new Error("connection refused"))).toBe(false); + expect(isRateLimitError("not an error")).toBe(false); + }); +}); + +describe("withRateLimitRetry", () => { + const noSleep = async () => {}; + + it("retries a rate-limited call and then succeeds", async () => { + let n = 0; + const out = await withRateLimitRetry(async () => { + if (n++ < 2) throw new Error("429 rate limit"); + return "ok"; + }, { sleep: noSleep }); + expect(out).toBe("ok"); + expect(n).toBe(3); + }); + + it("surfaces a non-rate-limit error immediately", async () => { + let n = 0; + await expect( + withRateLimitRetry(async () => { n++; throw new Error("bad prompt"); }, { sleep: noSleep }), + ).rejects.toThrow(/bad prompt/); + expect(n).toBe(1); + }); + + it("gives up after the retry budget", async () => { + let n = 0; + await expect( + withRateLimitRetry(async () => { n++; throw new Error("overloaded"); }, { retries: 2, sleep: noSleep }), + ).rejects.toThrow(/overloaded/); + expect(n).toBe(3); // initial + 2 retries + }); +}); From 9f4cc395b01dc474e244081d8359d5d5923b9ba4 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 16:25:43 +0000 Subject: [PATCH 30/39] fix: idempotent upsert in docs generate to stop duplicate rows --- src/commands/docs.ts | 2 +- src/docs/generate.ts | 22 ++++++--------- src/docs/index.ts | 2 +- src/docs/write.ts | 59 +++++++++++++++++++++++++++++++++++++++ tests/shared/docs.test.ts | 40 ++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 6691762b..b5628c7f 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -475,7 +475,7 @@ export async function runDocsCommand(args: string[]): Promise { const exclude = flagValues(args, "--exclude"); const limitRaw = flagValue(args, "--limit"); const limit = limitRaw === undefined ? undefined : Number(limitRaw); - const concurrency = Number(flagValue(args, "--concurrency") ?? "6"); + const concurrency = Number(flagValue(args, "--concurrency") ?? "4"); const project = flagValue(args, "--project") ?? ""; const snap = loadCurrentSnapshot(cwd); diff --git a/src/docs/generate.ts b/src/docs/generate.ts index 4b57a0b8..1d5903cb 100644 --- a/src/docs/generate.ts +++ b/src/docs/generate.ts @@ -18,7 +18,7 @@ import { buildAnchor, readSymbolSource } from "./anchors.js"; import { runPool } from "./pool.js"; -import { insertDocResilient, setDoc } from "./write.js"; +import { upsertDoc } from "./write.js"; import type { DocAnchor, QueryFn } from "./read.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -181,7 +181,7 @@ export async function generateDocs(args: GenerateArgs): Promise { if (args.limit !== undefined) targets = targets.slice(0, args.limit); const outcomes: GenOutcome[] = []; - await runPool(targets, args.concurrency ?? 6, async (t) => { + await runPool(targets, args.concurrency ?? 4, async (t) => { // Build the prompt context from current source; also pre-build anchors. const symInput: GenDocInput["symbols"] = []; const anchors: DocAnchor[] = []; @@ -217,18 +217,12 @@ export async function generateDocs(args: GenerateArgs): Promise { agent: args.agent ?? "docs-generate", plugin_version: args.pluginVersion, }; - // Fresh docs (the common bulk-generate case) go through insertDocResilient - // — a single INSERT, NOT setDoc's read-stability gate (~N reads). Under - // high concurrency the gate's read storm overloaded the backend and timed - // writes out. The resilient variant retries the INSERT on the backend's - // intermittent under-load timeouts (checking whether the row already - // landed before each retry, so it never forks a second v1). Only - // re-authored (--force) docs need setDoc's version-bump. - if (args.force && args.existing.has(t.doc_id)) { - await setDoc(args.query, args.tableName, write); - } else { - await insertDocResilient(args.query, args.tableName, write); - } + // Idempotent upsert keyed on the deterministic id = doc_id (DELETE-then- + // INSERT). Retrying a timed-out write can't fork a duplicate row — the + // retry deletes whatever landed and re-inserts exactly one. This replaced + // insertDocResilient, whose random-UUID INSERT + lag-prone read-back + // forked up to 4 rows per file under high-concurrency bulk generate. + await upsertDoc(args.query, args.tableName, write); outcomes.push({ doc_id: t.doc_id, status: "created" }); } catch (err) { outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `write failed: ${(err as Error).message}` }); diff --git a/src/docs/index.ts b/src/docs/index.ts index 31d6f4c7..5fa85ca3 100644 --- a/src/docs/index.ts +++ b/src/docs/index.ts @@ -6,7 +6,7 @@ * change for callers. */ -export { insertDoc, insertDocResilient, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; +export { insertDoc, insertDocResilient, upsertDoc, editDoc, setDoc, archiveDoc, _MAX_CONTENT_LENGTH } from "./write.js"; export type { InsertDocInput, EditDocInput, SetDocInput, WriteResult, ResilientWriteOpts } from "./write.js"; export { listDocs, listDocMeta, listDocsByIds, getDocLatest, parseAnchors } from "./read.js"; diff --git a/src/docs/write.ts b/src/docs/write.ts index 6d1ebe25..010ec704 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -205,6 +205,65 @@ export async function insertDocResilient( throw lastErr ?? new Error("insertDocResilient: exhausted retries"); } +/** + * Idempotent generate-write keyed on a DETERMINISTIC row id = doc_id. + * + * Bulk generate under high concurrency times writes out client-side while the + * backend often already committed; the old `insertDocResilient` then re-INSERTed + * (each with a fresh random UUID) because its read-back missed the landed row + * under read-after-write lag — forking up to 4 duplicate rows per file. + * + * This write is DELETE-then-INSERT on the fixed id = doc_id, so retrying the + * WHOLE op is safe: the retry deletes whatever landed and re-inserts exactly + * one row. One row per doc_id by construction — no random id, no read-back gate. + * Only client-side timeouts are retried; other errors surface immediately. + */ +export async function upsertDoc( + query: QueryFn, + tableName: string, + input: InsertDocInput, + opts: ResilientWriteOpts = {}, +): Promise { + assertValidContent(input.content); + if (input.doc_id.length === 0) throw new Error("Doc doc_id must not be empty"); + const safe = sqlIdent(tableName); + const id = input.doc_id; // deterministic: retries + re-runs target the same row + const anchors = serializeAnchors(input.anchors ?? []); + const tier: DocTier = input.tier ?? "fast"; + + const retries = opts.retries ?? WRITE_RETRIES; + const backoff = opts.backoffMs ?? WRITE_BACKOFF_MS; + const sleep = opts.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + + let lastErr: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const now = new Date().toISOString(); + // Clear any prior/partial row for this id first, then write exactly one. + await query(`DELETE FROM "${safe}" WHERE id = '${sqlStr(id)}'`); + const sql = + `INSERT INTO "${safe}" ` + + `(id, doc_id, path, content, anchors, tier, status, project, version, ` + + `created_at, updated_at, agent, plugin_version) ` + + `VALUES (` + + `'${sqlStr(id)}', '${sqlStr(input.doc_id)}', '${sqlStr(input.path)}', ` + + `E'${sqlStr(input.content)}', E'${sqlStr(anchors)}', '${sqlStr(tier)}', ` + + `'active', '${sqlStr(input.project ?? "")}', 1, ` + + `'${sqlStr(now)}', '${sqlStr(now)}', ` + + `'${sqlStr(input.agent ?? "manual")}', '${sqlStr(input.plugin_version ?? "")}'` + + `)`; + await query(sql); + return { doc_id: input.doc_id, version: 1 }; + } catch (err) { + if (!isTimeoutError(err)) throw err; + lastErr = err; + if (attempt === retries) break; + await sleep(backoff[Math.min(attempt, backoff.length - 1)]); + } + } + throw lastErr ?? new Error("upsertDoc: exhausted retries"); +} + /** * Edit an existing doc. Reads the latest version, then INSERTs a new row * with version+1 carrying the merged fields (omitted fields inherit from diff --git a/tests/shared/docs.test.ts b/tests/shared/docs.test.ts index 3b067fba..4730cea3 100644 --- a/tests/shared/docs.test.ts +++ b/tests/shared/docs.test.ts @@ -10,6 +10,7 @@ vi.mock("../../src/docs/stable-read.js", () => ({ import { insertDoc, insertDocResilient, + upsertDoc, editDoc, setDoc, archiveDoc, @@ -237,6 +238,45 @@ describe("insertDocResilient", () => { }); }); +// ── upsertDoc (idempotent generate-write — the duplicate-row fix) ───────────── + +describe("upsertDoc", () => { + const noSleep = async () => {}; + const timeout = () => { throw new Error("Query timeout after 10000ms"); }; + + it("DELETEs by deterministic id=doc_id then INSERTs exactly one row", async () => { + const { calls, query } = mockQuery([() => [], () => []]); + const res = await upsertDoc(query, TBL, { doc_id: "src/a.ts", path: "/docs/p/a.ts.md", content: "x", project: "p" }); + expect(res).toEqual({ doc_id: "src/a.ts", version: 1 }); + expect(calls).toHaveLength(2); + expect(calls[0]).toBe(`DELETE FROM "hivemind_docs" WHERE id = 'src/a.ts'`); + expect(calls[1]).toMatch(/^INSERT INTO "hivemind_docs"/); + // id column is the doc_id, NOT a random uuid + expect(calls[1]).toContain(`'src/a.ts', 'src/a.ts',`); + expect(calls[1]).toContain(", 1, "); // always version 1 + }); + + it("retry after a timeout re-runs DELETE+INSERT — never forks a second row", async () => { + // 1st DELETE ok, INSERT times out; retry: DELETE ok, INSERT ok. + const { calls, query } = mockQuery([() => [], timeout, () => [], () => []]); + const res = await upsertDoc(query, TBL, { doc_id: "src/a.ts", path: "/p", content: "x" }, { sleep: noSleep }); + expect(res).toEqual({ doc_id: "src/a.ts", version: 1 }); + // exactly one INSERT succeeds; the DELETE on retry guarantees single-row. + const inserts = calls.filter(c => c.startsWith("INSERT INTO")); + const deletes = calls.filter(c => c.startsWith("DELETE FROM")); + expect(inserts.length).toBeGreaterThanOrEqual(1); + expect(deletes.length).toBe(inserts.length); // every INSERT preceded by a DELETE + }); + + it("surfaces a non-timeout error immediately (no retry)", async () => { + const { calls, query } = mockQuery([() => [], () => { throw new Error("403 forbidden"); }]); + await expect( + upsertDoc(query, TBL, { doc_id: "src/a.ts", path: "/p", content: "x" }, { sleep: noSleep }), + ).rejects.toThrow(/403/); + expect(calls).toHaveLength(2); // DELETE + the failing INSERT, no retry + }); +}); + // ── editDoc ─────────────────────────────────────────────────────────────────── describe("editDoc", () => { From 1e9ae9e909064fd2d6c232a5e5c01e9c063da989 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 16:50:50 +0000 Subject: [PATCH 31/39] feat: batch docs generation to amortize per-call llm boot --- src/commands/docs.ts | 10 +- src/docs/generate.ts | 157 ++++++++++++++++++++++++----- src/docs/refresh-llm.ts | 23 ++++- tests/shared/docs-generate.test.ts | 103 ++++++++++++++++++- 4 files changed, 263 insertions(+), 30 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index b5628c7f..91b265b5 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -47,7 +47,7 @@ import { type DocTier, type DocAnchor, } from "../docs/index.js"; -import { makeHostGenerate, makeHostGenerateDoc } from "../docs/refresh-llm.js"; +import { makeHostGenerate, makeHostGenerateDoc, makeHostBatchGenerateDoc } from "../docs/refresh-llm.js"; import { generateDocs, selectTargets, type GenScope } from "../docs/generate.js"; import { loadCurrentSnapshot } from "../graph/load-current.js"; import { isMissingTableError } from "../deeplake-schema.js"; @@ -141,7 +141,7 @@ function parseLimit(args: string[]): number { return n; } -const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor", "--scope", "--include", "--exclude", "--concurrency", "--force"]); +const KNOWN_FLAGS = new Set(["--file", "--project", "--tier", "--path", "--status", "--limit", "--cwd", "--dry-run", "--anchor", "--scope", "--include", "--exclude", "--concurrency", "--force", "--batch"]); /** Flags that take NO value — they must not consume the following token. */ const BOOLEAN_FLAGS = new Set(["--dry-run", "--force"]); @@ -503,10 +503,14 @@ export async function runDocsCommand(args: string[]): Promise { } await api.ensureDocsTable(tableName); + const batchSize = Number(flagValue(args, "--batch") ?? "1"); const report = await generateDocs({ query, tableName, snap, repoRoot: cwd, project, scope, include, exclude, existing, force, limit, concurrency, - generate: makeHostGenerateDoc(), agent: cfg.userName, pluginVersion, + generate: makeHostGenerateDoc(), + batchSize, + batchGenerate: batchSize > 1 ? makeHostBatchGenerateDoc() : undefined, + agent: cfg.userName, pluginVersion, }); console.log(`Generated ${report.created}, skipped ${report.skipped}, failed ${report.failed} (of ${report.targets} targets).`); for (const o of report.outcomes) { diff --git a/src/docs/generate.ts b/src/docs/generate.ts index 1d5903cb..481aef06 100644 --- a/src/docs/generate.ts +++ b/src/docs/generate.ts @@ -53,6 +53,19 @@ export interface GenDocInput { } export type GenerateDocFn = (input: GenDocInput) => Promise; +/** + * Batched generator: document K files in ONE host-LLM call and return a + * doc_id → markdown map. The per-file `claude -p` boot (~15s) dominates a single + * doc; batching amortizes it across K files (~3.7x faster measured). Anchors are + * still computed per file from the graph — only the prose is batched. Any file + * the model omits is simply absent from the map, so the caller falls back to a + * single-file call for it. + */ +export type BatchGenerateFn = (inputs: GenDocInput[]) => Promise>; + +/** Machine marker the model emits before each file's doc (robust to split). */ +const BATCH_MARKER_RE = /<<>>[ \t]*\n?/; + /** Convert a glob (`*`, `**`, `?`) to an anchored RegExp over forward-slash paths. */ export function globToRegExp(glob: string): RegExp { let re = ""; @@ -137,6 +150,59 @@ export function buildGeneratePrompt(input: GenDocInput): string { ].join("\n"); } +/** Render the symbols block for one file (shared with the batch prompt). */ +function renderSymbols(input: GenDocInput): string { + return input.symbols + .map((s) => `### ${s.id}\n${s.signature ? s.signature + "\n" : ""}\n\`\`\`\n${s.source}\n\`\`\``) + .join("\n\n") || "(none)"; +} + +/** + * Build ONE prompt documenting several files. The model must prefix each file's + * doc with an exact machine marker `<<>>` so the response splits + * back apart deterministically — see {@link parseBatchDocs}. + */ +export function buildBatchGeneratePrompt(inputs: GenDocInput[]): string { + const blocks = inputs.map((input) => + [`<<>>`, `## File: ${input.file}`, "", "## Symbols", renderSymbols(input)].join("\n"), + ); + return [ + "You are writing concise internal documentation for MULTIPLE source files.", + "For EACH file below, output its doc PREFIXED by a line EXACTLY of the form:", + "<<>>", + "using the exact path shown for that file, then the markdown doc (what the file", + "is for + one short line per key symbol, under ~1200 characters).", + "Output ONLY these marker+doc sections, in order. No preamble, no outer code fence.", + "", + "=== FILES ===", + "", + blocks.join("\n\n----------\n\n"), + ].join("\n"); +} + +/** + * Split a batched response into doc_id → markdown. Keys by the marker path, + * matched against the requested `inputs` (so only real files map back). A file + * whose marker is missing is simply absent — the caller regenerates it singly. + */ +export function parseBatchDocs(response: string, inputs: GenDocInput[]): Map { + const wanted = new Map(inputs.map((i) => [i.file, i.doc_id])); + const out = new Map(); + const marker = new RegExp(BATCH_MARKER_RE.source, "g"); + const matches = [...response.matchAll(marker)]; + for (let i = 0; i < matches.length; i++) { + const m = matches[i]; + const file = m[1].trim(); + const docId = wanted.get(file); + if (!docId) continue; // unknown/hallucinated path + const start = m.index! + m[0].length; + const end = i + 1 < matches.length ? matches[i + 1].index! : response.length; + const body = response.slice(start, end).trim(); + if (body) out.set(docId, body); + } + return out; +} + export interface GenerateArgs { query: QueryFn; tableName: string; @@ -152,6 +218,10 @@ export interface GenerateArgs { limit?: number; concurrency?: number; generate: GenerateDocFn; + /** When >1 with `batchGenerate`, document this many files per LLM call. */ + batchSize?: number; + /** Batched generator (documents K files at once). Falls back to `generate`. */ + batchGenerate?: BatchGenerateFn; agent?: string; pluginVersion?: string; } @@ -181,8 +251,10 @@ export async function generateDocs(args: GenerateArgs): Promise { if (args.limit !== undefined) targets = targets.slice(0, args.limit); const outcomes: GenOutcome[] = []; - await runPool(targets, args.concurrency ?? 4, async (t) => { - // Build the prompt context from current source; also pre-build anchors. + + // Prep a target into its prompt input + per-file anchors (pure, no LLM). + // Returns null (and records a skip) when nothing can be anchored. + const prep = (t: GenTarget): { input: GenDocInput; anchors: DocAnchor[] } | null => { const symInput: GenDocInput["symbols"] = []; const anchors: DocAnchor[] = []; for (const n of t.symbols) { @@ -193,41 +265,76 @@ export async function generateDocs(args: GenerateArgs): Promise { } if (anchors.length === 0) { outcomes.push({ doc_id: t.doc_id, status: "skipped", reason: "no readable symbols to anchor" }); - return; - } - let content: string; - try { - content = await args.generate({ doc_id: t.doc_id, file: t.file, symbols: symInput }); - } catch (err) { - outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `generate failed: ${(err as Error).message}` }); - return; + return null; } + return { input: { doc_id: t.doc_id, file: t.file, symbols: symInput }, anchors }; + }; + + // Persist one doc via the idempotent upsert (deterministic id = doc_id, so a + // retried write can never fork a duplicate row). + const writeDoc = async (docId: string, content: string, anchors: DocAnchor[]): Promise => { if (content.trim() === "") { - outcomes.push({ doc_id: t.doc_id, status: "failed", reason: "empty content" }); + outcomes.push({ doc_id: docId, status: "failed", reason: "empty content" }); return; } try { - const write = { - doc_id: t.doc_id, - path: defaultVfsPath(project, t.doc_id), + await upsertDoc(args.query, args.tableName, { + doc_id: docId, + path: defaultVfsPath(project, docId), content, anchors, - tier: "fast" as const, + tier: "fast", project, agent: args.agent ?? "docs-generate", plugin_version: args.pluginVersion, - }; - // Idempotent upsert keyed on the deterministic id = doc_id (DELETE-then- - // INSERT). Retrying a timed-out write can't fork a duplicate row — the - // retry deletes whatever landed and re-inserts exactly one. This replaced - // insertDocResilient, whose random-UUID INSERT + lag-prone read-back - // forked up to 4 rows per file under high-concurrency bulk generate. - await upsertDoc(args.query, args.tableName, write); - outcomes.push({ doc_id: t.doc_id, status: "created" }); + }); + outcomes.push({ doc_id: docId, status: "created" }); } catch (err) { - outcomes.push({ doc_id: t.doc_id, status: "failed", reason: `write failed: ${(err as Error).message}` }); + outcomes.push({ doc_id: docId, status: "failed", reason: `write failed: ${(err as Error).message}` }); } - }); + }; + + const genSingle = async (input: GenDocInput, anchors: DocAnchor[]): Promise => { + let content: string; + try { + content = await args.generate(input); + } catch (err) { + outcomes.push({ doc_id: input.doc_id, status: "failed", reason: `generate failed: ${(err as Error).message}` }); + return; + } + await writeDoc(input.doc_id, content, anchors); + }; + + const concurrency = args.concurrency ?? 4; + const batchSize = args.batchSize ?? 1; + + if (batchSize > 1 && args.batchGenerate) { + // Batched: document `batchSize` files per LLM call (amortizes the per-call + // boot). Anchors stay per-file; any file the model omits from the batch + // response falls back to a single-file call so coverage is never lost. + const prepped = targets.map(prep).filter((p): p is { input: GenDocInput; anchors: DocAnchor[] } => p !== null); + const batches: Array = []; + for (let i = 0; i < prepped.length; i += batchSize) batches.push(prepped.slice(i, i + batchSize)); + const batchGen = args.batchGenerate; + await runPool(batches, concurrency, async (batch) => { + let map: Map; + try { + map = await batchGen(batch.map((b) => b.input)); + } catch { + map = new Map(); // whole-batch failure → everyone falls back to single + } + for (const b of batch) { + const content = map.get(b.input.doc_id); + if (content && content.trim() !== "") await writeDoc(b.input.doc_id, content, b.anchors); + else await genSingle(b.input, b.anchors); // omitted by the model → single + } + }); + } else { + await runPool(targets, concurrency, async (t) => { + const p = prep(t); + if (p) await genSingle(p.input, p.anchors); + }); + } return { outcomes, diff --git a/src/docs/refresh-llm.ts b/src/docs/refresh-llm.ts index 7e200e90..0d4a5108 100644 --- a/src/docs/refresh-llm.ts +++ b/src/docs/refresh-llm.ts @@ -16,7 +16,14 @@ import { execFileSync } from "node:child_process"; import { buildClaudeInvocation, buildTrailingPromptInvocation, type ClaudeInvocation } from "../hooks/wiki-worker-spawn.js"; import { resolveCliBin } from "../utils/resolve-cli-bin.js"; import { buildRefreshPrompt, type GenerateFn } from "./refresh.js"; -import { buildGeneratePrompt, type GenerateDocFn } from "./generate.js"; +import { + buildGeneratePrompt, + buildBatchGeneratePrompt, + parseBatchDocs, + type GenerateDocFn, + type BatchGenerateFn, + type GenDocInput, +} from "./generate.js"; /** * Defensively unwrap the model's output. The prompt asks for raw markdown, @@ -102,6 +109,20 @@ export function makeHostGenerateDoc(timeoutMs = 120_000, env: NodeJS.ProcessEnv return async (input) => runHostPrompt(spec, bin, buildGeneratePrompt(input), timeoutMs); } +/** + * BATCHED generation backed by the resolved host agent: one CLI call documents + * K files. Amortizes the ~15s per-call boot across the batch (~3.7x faster). + * A longer default timeout accounts for the larger prompt + K docs of output. + */ +export function makeHostBatchGenerateDoc(timeoutMs = 240_000, env: NodeJS.ProcessEnv = process.env): BatchGenerateFn { + const spec = resolveDocLlmSpec(env); + const bin = resolveCliBin(spec.bin); + return async (inputs: GenDocInput[]) => { + const raw = runHostPrompt(spec, bin, buildBatchGeneratePrompt(inputs), timeoutMs); + return parseBatchDocs(raw, inputs); + }; +} + /** Run a single prompt through the host `claude` CLI and return the unwrapped output. */ export function runClaudePrompt(bin: string, prompt: string, timeoutMs = 120_000): string { return runHostPrompt(REGISTRY.claude, bin, prompt, timeoutMs); diff --git a/tests/shared/docs-generate.test.ts b/tests/shared/docs-generate.test.ts index 799f7a36..2d11fe0b 100644 --- a/tests/shared/docs-generate.test.ts +++ b/tests/shared/docs-generate.test.ts @@ -9,7 +9,8 @@ vi.mock("../../src/docs/stable-read.js", () => ({ })); import { normalizeForHash, hashSource } from "../../src/docs/anchors.js"; -import { globToRegExp, selectTargets, generateDocs, DEFAULT_EXCLUDE_GLOBS } from "../../src/docs/generate.js"; +import { globToRegExp, selectTargets, generateDocs, DEFAULT_EXCLUDE_GLOBS, buildBatchGeneratePrompt, parseBatchDocs } from "../../src/docs/generate.js"; +import type { GenDocInput } from "../../src/docs/generate.js"; import type { GraphNode, GraphSnapshot } from "../../src/graph/types.js"; function node(id: string, file: string, loc: string, kind: GraphNode["kind"] = "function"): GraphNode { @@ -150,3 +151,103 @@ describe("generateDocs", () => { expect(report.outcomes[0].reason).toMatch(/LLM down/); }); }); + +// ── batched generation (parser + prompt) ────────────────────────────────────── + +const bIn = (docId: string): GenDocInput => ({ doc_id: docId, file: docId, symbols: [{ id: `${docId}:f`, source: "x" }] }); + +describe("buildBatchGeneratePrompt", () => { + it("emits an exact marker + File block for every input file", () => { + const p = buildBatchGeneratePrompt([bIn("src/a.ts"), bIn("src/b.ts")]); + expect(p).toContain("<<>>"); + expect(p).toContain("<<>>"); + expect(p).toContain("## File: src/a.ts"); + expect(p).toContain("## File: src/b.ts"); + }); +}); + +describe("parseBatchDocs", () => { + const inputs = [bIn("src/a.ts"), bIn("src/b.ts"), bIn("src/c.ts")]; + + it("splits a marked response into doc_id -> content", () => { + const resp = [ + "<<>>", "# A", "does A", + "<<>>", "# B", "does B", + "<<>>", "# C", + ].join("\n"); + const m = parseBatchDocs(resp, inputs); + expect(m.size).toBe(3); + expect(m.get("src/a.ts")).toBe("# A\ndoes A"); + expect(m.get("src/b.ts")).toBe("# B\ndoes B"); + expect(m.get("src/c.ts")).toBe("# C"); + }); + + it("omits a file the model dropped (caller falls back to single)", () => { + const resp = "<<>>\n# A\n<<>>\n# C"; + const m = parseBatchDocs(resp, inputs); + expect(m.has("src/b.ts")).toBe(false); // dropped → absent + expect([...m.keys()].sort()).toEqual(["src/a.ts", "src/c.ts"]); + }); + + it("ignores a hallucinated path not in the batch", () => { + const resp = "<<>>\n# A\n<<>>\n# nope"; + const m = parseBatchDocs(resp, inputs); + expect(m.size).toBe(1); + expect(m.get("src/a.ts")).toBe("# A"); + }); + + it("drops an empty body", () => { + const resp = "<<>>\n \n<<>>\n# B"; + const m = parseBatchDocs(resp, inputs); + expect(m.has("src/a.ts")).toBe(false); + expect(m.get("src/b.ts")).toBe("# B"); + }); +}); + +// ── generateDocs BATCH path (fallback on omitted files) ─────────────────────── + +describe("generateDocs — batch path", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "docs-batch-")); + writeFileSync(join(dir, "a.ts"), "export function foo() {\n return 1;\n}\n"); + writeFileSync(join(dir, "b.ts"), "export function bar() {\n return 2;\n}\n"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + const s = () => snap([ + node("a.ts:foo:function", "a.ts", "L1-L3"), + node("b.ts:bar:function", "b.ts", "L1-L3"), + ]); + + it("uses the batch generator, and falls back to single for files it omits", async () => { + const { query } = mockQuery([]); + // batch returns ONLY a.ts → b.ts must fall back to the single generator. + const batchGenerate = vi.fn(async (inputs: GenDocInput[]) => { + const m = new Map(); + for (const i of inputs) if (i.doc_id === "a.ts") m.set("a.ts", "# a batched"); + return m; + }); + const generate = vi.fn(async (i: GenDocInput) => `# ${i.doc_id} single`); + const report = await generateDocs({ + query, tableName: "hivemind_docs", snap: s(), repoRoot: dir, + existing: new Set(), generate, batchSize: 5, batchGenerate, concurrency: 1, + }); + expect(report.created).toBe(2); // both landed + expect(batchGenerate).toHaveBeenCalledOnce(); // one batch call for both + expect(generate).toHaveBeenCalledOnce(); // only the omitted b.ts + expect(generate.mock.calls[0][0].doc_id).toBe("b.ts"); + }); + + it("falls back to single for the whole batch when the batch call throws", async () => { + const { query } = mockQuery([]); + const batchGenerate = vi.fn(async () => { throw new Error("batch LLM down"); }); + const generate = vi.fn(async (i: GenDocInput) => `# ${i.doc_id}`); + const report = await generateDocs({ + query, tableName: "hivemind_docs", snap: s(), repoRoot: dir, + existing: new Set(), generate, batchSize: 5, batchGenerate, concurrency: 1, + }); + expect(report.created).toBe(2); + expect(generate).toHaveBeenCalledTimes(2); // both fell back + }); +}); From 6a9f6bfe2b01abea526ae1c1ff16f0abf50d982e Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 19:14:33 +0000 Subject: [PATCH 32/39] feat: batch docs generate by default and raise write timeout --- src/commands/docs.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 91b265b5..a945d39d 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -69,7 +69,8 @@ Usage: impacted docs without calling the LLM or writing anything. hivemind docs generate [--cwd ] [--scope file|symbol] [--include ] [--exclude ] [--limit N] [--concurrency N] - [--force] [--dry-run] + [--batch N] [--force] [--dry-run] + Batches 5 files per LLM call by default (~2.5x faster); --batch 1 to opt out. Auto-author docs for the codebase from the AST graph (which already skips .gitignored / non-code files). Default scope=file (one doc per file, anchored to its symbols). Skips files that already have a doc unless @@ -503,7 +504,13 @@ export async function runDocsCommand(args: string[]): Promise { } await api.ensureDocsTable(tableName); - const batchSize = Number(flagValue(args, "--batch") ?? "1"); + // Bulk generate writes under load need a longer client timeout than the 10s + // default, or writes abort mid-commit and drop files. Scope it to this + // command (not global reads); the user can still override via env. + if (!process.env.HIVEMIND_QUERY_TIMEOUT_MS) process.env.HIVEMIND_QUERY_TIMEOUT_MS = "30000"; + // Batch by default (5 files/call) — amortizes the per-call LLM boot ~2.5x. + // `--batch 1` opts out; larger batches trade a little quality for speed. + const batchSize = Number(flagValue(args, "--batch") ?? "5"); const report = await generateDocs({ query, tableName, snap, repoRoot: cwd, project, scope, include, exclude, existing, force, limit, concurrency, From efdb78e7974650130638c67e769f619c66ea27f3 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 20:21:53 +0000 Subject: [PATCH 33/39] feat: compute and store a search embedding per doc --- src/deeplake-schema.ts | 4 +++ src/docs/embed.ts | 56 ++++++++++++++++++++++++++++++++++++++++++ src/docs/generate.ts | 6 +++++ src/docs/refresh.ts | 5 ++++ src/docs/write.ts | 22 ++++++++++++++--- 5 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 src/docs/embed.ts diff --git a/src/deeplake-schema.ts b/src/deeplake-schema.ts index 8d20a9f4..2b483f94 100644 --- a/src/deeplake-schema.ts +++ b/src/deeplake-schema.ts @@ -205,6 +205,10 @@ export const DOCS_COLUMNS: readonly ColumnDef[] = Object.freeze([ { name: "updated_at", sql: "TEXT NOT NULL DEFAULT ''" }, { name: "agent", sql: "TEXT NOT NULL DEFAULT 'manual'" }, { name: "plugin_version", sql: "TEXT NOT NULL DEFAULT ''" }, + // Semantic-search vector over `content` (nomic, DOC_PREFIX). Nullable/empty + // when embeddings are off or not yet backfilled — `docs/find/` guards with + // ARRAY_LENGTH(...) > 0, exactly like grep-core does for summaries. + { name: "content_embedding", sql: "FLOAT4[]" }, ]); // ── Module-load lint ──────────────────────────────────────────────────────── diff --git a/src/docs/embed.ts b/src/docs/embed.ts new file mode 100644 index 00000000..c8bed2b3 --- /dev/null +++ b/src/docs/embed.ts @@ -0,0 +1,56 @@ +/** + * Doc-content embedder — the search vector for `content_embedding`. + * + * Mirrors the capture-hook pattern (src/hooks/capture.ts): one nomic embedding + * per doc via the shared daemon, `document` kind (applies DOC_PREFIX). Reuses a + * single `EmbedClient` across a bulk generate run. Best-effort and null-safe — + * disabled embeddings or any daemon failure yield `null`, which lands as a NULL + * `content_embedding` (lexical `docs/find/` still works; semantic is guarded by + * `ARRAY_LENGTH(content_embedding,1) > 0`). NEVER blocks or fails a doc write. + */ + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; + +/** A best-effort text → vector function; returns null when unavailable. */ +export type DocEmbedder = (text: string) => Promise; + +function resolveEmbedDaemonPath(): string { + return join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); +} + +/** + * Build a reusable doc embedder. When embeddings are globally disabled, returns + * a no-op that always yields null (no daemon round-trip). Otherwise reuses one + * `EmbedClient` and swallows failures to null. + */ +export function makeDocEmbedder(): DocEmbedder { + if (embeddingsDisabled()) return async () => null; + const client = new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }); + return async (text: string) => { + try { + return await client.embed(text, "document"); + } catch { + return null; + } + }; +} + +/** + * Query-side embedder for `docs/find/` (kind='query' → QUERY_PREFIX, the nomic + * asymmetric search convention). Null when embeddings are disabled/unreachable + * → `docs/find/` degrades to lexical search. + */ +export function makeQueryEmbedder(): DocEmbedder { + if (embeddingsDisabled()) return async () => null; + const client = new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }); + return async (text: string) => { + try { + return await client.embed(text, "query"); + } catch { + return null; + } + }; +} diff --git a/src/docs/generate.ts b/src/docs/generate.ts index 481aef06..c111469c 100644 --- a/src/docs/generate.ts +++ b/src/docs/generate.ts @@ -19,6 +19,7 @@ import { buildAnchor, readSymbolSource } from "./anchors.js"; import { runPool } from "./pool.js"; import { upsertDoc } from "./write.js"; +import type { DocEmbedder } from "./embed.js"; import type { DocAnchor, QueryFn } from "./read.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -222,6 +223,8 @@ export interface GenerateArgs { batchSize?: number; /** Batched generator (documents K files at once). Falls back to `generate`. */ batchGenerate?: BatchGenerateFn; + /** Optional embedder for the doc search vector (best-effort, null-safe). */ + embed?: DocEmbedder; agent?: string; pluginVersion?: string; } @@ -278,6 +281,8 @@ export async function generateDocs(args: GenerateArgs): Promise { return; } try { + // Best-effort search vector; null → NULL column, never blocks the write. + const content_embedding = args.embed ? (await args.embed(content)) ?? undefined : undefined; await upsertDoc(args.query, args.tableName, { doc_id: docId, path: defaultVfsPath(project, docId), @@ -287,6 +292,7 @@ export async function generateDocs(args: GenerateArgs): Promise { project, agent: args.agent ?? "docs-generate", plugin_version: args.pluginVersion, + content_embedding, }); outcomes.push({ doc_id: docId, status: "created" }); } catch (err) { diff --git a/src/docs/refresh.ts b/src/docs/refresh.ts index f3b89733..9b55dc5c 100644 --- a/src/docs/refresh.ts +++ b/src/docs/refresh.ts @@ -20,6 +20,7 @@ import { buildAnchor, readSymbolSource } from "./anchors.js"; import { gateDocEdit, type GateResult } from "./gate.js"; import { runPool, withRateLimitRetry } from "./pool.js"; import { archiveDoc, setDoc } from "./write.js"; +import type { DocEmbedder } from "./embed.js"; import type { DocAnchor, DocRow, QueryFn } from "./read.js"; import type { ImpactedDoc, StaleReason } from "./impact.js"; import type { GraphNode, GraphSnapshot } from "../graph/types.js"; @@ -73,6 +74,8 @@ export interface RefreshArgs { maxChangedLines?: number; /** Max docs rewritten in parallel. Default 4. */ concurrency?: number; + /** Optional embedder to refresh the doc search vector (best-effort). */ + embed?: DocEmbedder; } /** Build the prompt for one doc refresh — bounded-edit, freshness-focused. */ @@ -213,6 +216,7 @@ export async function refreshDocs(args: RefreshArgs): Promise { return; } + const content_embedding = args.embed ? (await args.embed(newContent)) ?? undefined : undefined; const res = await setDoc(args.query, args.tableName, { doc_id: doc.doc_id, path: doc.path, @@ -222,6 +226,7 @@ export async function refreshDocs(args: RefreshArgs): Promise { project: doc.project, agent: args.agent ?? "docs-refresh", plugin_version: args.pluginVersion, + content_embedding, }); outcomes.push({ doc_id: imp.doc_id, status: "refreshed", version: res.version }); }); diff --git a/src/docs/write.ts b/src/docs/write.ts index 010ec704..335f1ec5 100644 --- a/src/docs/write.ts +++ b/src/docs/write.ts @@ -21,6 +21,7 @@ import { randomUUID } from "node:crypto"; import { sqlIdent, sqlStr } from "../utils/sql.js"; +import { embeddingSqlLiteral } from "../embeddings/sql.js"; import type { DocAnchor, DocRow, DocTier, QueryFn } from "./read.js"; import { getDocLatest } from "./read.js"; @@ -41,6 +42,8 @@ export interface InsertDocInput { agent?: string; /** Plugin version that produced the write. Empty string lands the default. */ plugin_version?: string; + /** Optional precomputed nomic embedding of `content` (search vector). */ + content_embedding?: number[]; } export interface SetDocInput { @@ -60,6 +63,8 @@ export interface SetDocInput { status?: "active" | "archived"; agent?: string; plugin_version?: string; + /** Optional precomputed nomic embedding of `content` (search vector). */ + content_embedding?: number[]; } export interface EditDocInput { @@ -79,6 +84,8 @@ export interface EditDocInput { project?: string; agent?: string; plugin_version?: string; + /** Optional precomputed nomic embedding of `content` (search vector). */ + content_embedding?: number[]; } export interface WriteResult { @@ -128,7 +135,7 @@ export async function insertDoc( const sql = `INSERT INTO "${safe}" ` + `(id, doc_id, path, content, anchors, tier, status, project, version, ` + - `created_at, updated_at, agent, plugin_version) ` + + `created_at, updated_at, agent, plugin_version, content_embedding) ` + `VALUES (` + `'${sqlStr(rowId)}', ` + `'${sqlStr(input.doc_id)}', ` + @@ -142,7 +149,8 @@ export async function insertDoc( `'${sqlStr(now)}', ` + `'${sqlStr(now)}', ` + `'${sqlStr(input.agent ?? "manual")}', ` + - `'${sqlStr(input.plugin_version ?? "")}'` + + `'${sqlStr(input.plugin_version ?? "")}', ` + + `${embeddingSqlLiteral(input.content_embedding)}` + `)`; await query(sql); return { doc_id: input.doc_id, version: 1 }; @@ -244,13 +252,14 @@ export async function upsertDoc( const sql = `INSERT INTO "${safe}" ` + `(id, doc_id, path, content, anchors, tier, status, project, version, ` + - `created_at, updated_at, agent, plugin_version) ` + + `created_at, updated_at, agent, plugin_version, content_embedding) ` + `VALUES (` + `'${sqlStr(id)}', '${sqlStr(input.doc_id)}', '${sqlStr(input.path)}', ` + `E'${sqlStr(input.content)}', E'${sqlStr(anchors)}', '${sqlStr(tier)}', ` + `'active', '${sqlStr(input.project ?? "")}', 1, ` + `'${sqlStr(now)}', '${sqlStr(now)}', ` + - `'${sqlStr(input.agent ?? "manual")}', '${sqlStr(input.plugin_version ?? "")}'` + + `'${sqlStr(input.agent ?? "manual")}', '${sqlStr(input.plugin_version ?? "")}', ` + + `${embeddingSqlLiteral(input.content_embedding)}` + `)`; await query(sql); return { doc_id: input.doc_id, version: 1 }; @@ -308,6 +317,7 @@ export async function setDoc( project: input.project, agent: input.agent, plugin_version: input.plugin_version, + content_embedding: input.content_embedding, }); } return updateInPlace(query, tableName, previous, { @@ -320,6 +330,7 @@ export async function setDoc( project: input.project, agent: input.agent, plugin_version: input.plugin_version, + content_embedding: input.content_embedding, }); } @@ -382,6 +393,9 @@ async function updateInPlace( `tier = '${sqlStr(tier)}', ` + `status = '${sqlStr(status)}', ` + `project = '${sqlStr(project)}', ` + + // Only rewrite the search vector when a fresh one is supplied — a + // status-only edit must NOT null out the existing embedding. + `${next.content_embedding !== undefined ? `content_embedding = ${embeddingSqlLiteral(next.content_embedding)}, ` : ""}` + `version = ${nextVersion}, ` + `updated_at = '${sqlStr(now)}', ` + `agent = '${sqlStr(next.agent ?? "manual")}', ` + From 09bec5c03e0049c34339a7b8390ea72aaf5cc3e6 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 20:22:05 +0000 Subject: [PATCH 34/39] feat: semantic docs search via docs/find plus reindex backfill --- src/commands/docs.ts | 22 +++++++++ src/docs/backfill.ts | 44 ++++++++++++++++++ src/docs/vfs-handler.ts | 33 ++++++++++++++ src/graph/session-context.ts | 6 +++ src/hooks/pre-tool-use.ts | 3 +- src/shell/grep-core.ts | 66 +++++++++++++++++++++++++++ tests/claude-code/grep-core.test.ts | 42 +++++++++++++++++ tests/shared/docs-backfill.test.ts | 37 +++++++++++++++ tests/shared/docs-vfs-handler.test.ts | 45 ++++++++++++++++++ 9 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 src/docs/backfill.ts create mode 100644 tests/shared/docs-backfill.test.ts diff --git a/src/commands/docs.ts b/src/commands/docs.ts index a945d39d..234376eb 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -49,6 +49,8 @@ import { } from "../docs/index.js"; import { makeHostGenerate, makeHostGenerateDoc, makeHostBatchGenerateDoc } from "../docs/refresh-llm.js"; import { generateDocs, selectTargets, type GenScope } from "../docs/generate.js"; +import { makeDocEmbedder } from "../docs/embed.js"; +import { backfillDocEmbeddings } from "../docs/backfill.js"; import { loadCurrentSnapshot } from "../graph/load-current.js"; import { isMissingTableError } from "../deeplake-schema.js"; @@ -71,6 +73,9 @@ Usage: [--exclude ] [--limit N] [--concurrency N] [--batch N] [--force] [--dry-run] Batches 5 files per LLM call by default (~2.5x faster); --batch 1 to opt out. + hivemind docs reindex + Backfill semantic-search vectors for docs that lack them (no LLM, embed + daemon only). Run once after enabling embeddings on an existing corpus. Auto-author docs for the codebase from the AST graph (which already skips .gitignored / non-code files). Default scope=file (one doc per file, anchored to its symbols). Skips files that already have a doc unless @@ -420,6 +425,7 @@ export async function runDocsCommand(args: string[]): Promise { impacted, docsById, generate: makeHostGenerate(), + embed: makeDocEmbedder(), agent: cfg.userName, pluginVersion, }); @@ -448,6 +454,7 @@ export async function runDocsCommand(args: string[]): Promise { include: changed, existing, generate: makeHostGenerateDoc(), + embed: makeDocEmbedder(), agent: cfg.userName, pluginVersion, }); @@ -461,6 +468,20 @@ export async function runDocsCommand(args: string[]): Promise { return; } + if (sub === "reindex") { + // Backfill content_embedding for docs missing it (no LLM — embed daemon only). + if (!process.env.HIVEMIND_QUERY_TIMEOUT_MS) process.env.HIVEMIND_QUERY_TIMEOUT_MS = "30000"; + try { + await api.ensureDocsTable(tableName); + const report = await backfillDocEmbeddings(query, tableName, makeDocEmbedder()); + console.log(`Reindexed: ${report.embedded} embedded (of ${report.scanned} active docs; ${report.skipped} already had a vector or skipped).`); + } catch (err) { + if (!isMissingTableError((err as Error).message)) throw err; + console.log("(no docs table yet — nothing to reindex)"); + } + return; + } + if (sub === "generate") { const cwd = flagValue(args, "--cwd") ?? process.cwd(); const dryRun = args.includes("--dry-run"); @@ -517,6 +538,7 @@ export async function runDocsCommand(args: string[]): Promise { generate: makeHostGenerateDoc(), batchSize, batchGenerate: batchSize > 1 ? makeHostBatchGenerateDoc() : undefined, + embed: makeDocEmbedder(), agent: cfg.userName, pluginVersion, }); console.log(`Generated ${report.created}, skipped ${report.skipped}, failed ${report.failed} (of ${report.targets} targets).`); diff --git a/src/docs/backfill.ts b/src/docs/backfill.ts new file mode 100644 index 00000000..99716fa4 --- /dev/null +++ b/src/docs/backfill.ts @@ -0,0 +1,44 @@ +/** + * Backfill `content_embedding` for docs that lack it — the migration path for + * docs generated before semantic search existed (or while embeddings were off). + * Cheap: NO LLM calls, only the embed daemon over existing `content`. Idempotent + * — skips docs that already carry a non-empty vector. + */ + +import { embeddingSqlLiteral } from "../embeddings/sql.js"; +import { sqlIdent, sqlStr } from "../utils/sql.js"; +import { runPool } from "./pool.js"; +import type { QueryFn } from "./read.js"; +import type { DocEmbedder } from "./embed.js"; + +export interface BackfillReport { + scanned: number; + embedded: number; + /** Already had a vector (or generation failed / embedder off). */ + skipped: number; +} + +export async function backfillDocEmbeddings( + query: QueryFn, + tableName: string, + embed: DocEmbedder, + concurrency = 4, +): Promise { + const safe = sqlIdent(tableName); + const rows = await query( + `SELECT id, content, ARRAY_LENGTH(content_embedding, 1) AS dims ` + + `FROM "${safe}" WHERE status = 'active'`, + ); + const missing = rows.filter((r) => r["dims"] == null || Number(r["dims"]) === 0); + let embedded = 0; + await runPool(missing, concurrency, async (r) => { + const vec = await embed(String(r["content"] ?? "")); + if (!vec || vec.length === 0) return; // embedder off / failed → leave NULL + await query( + `UPDATE "${safe}" SET content_embedding = ${embeddingSqlLiteral(vec)} ` + + `WHERE id = '${sqlStr(String(r["id"]))}'`, + ); + embedded++; + }); + return { scanned: rows.length, embedded, skipped: rows.length - embedded }; +} diff --git a/src/docs/vfs-handler.ts b/src/docs/vfs-handler.ts index 6cb0eb24..e4813c15 100644 --- a/src/docs/vfs-handler.ts +++ b/src/docs/vfs-handler.ts @@ -21,19 +21,52 @@ import { listDocMeta, listDocsByIds, getDocLatest, type QueryFn } from "./read.js"; import { buildDocsIndex, dirOf, firstDocLine, type DocMeta } from "./index-render.js"; +import { searchDocs, type SearchOptions } from "../shell/grep-core.js"; +import { sqlLike } from "../utils/sql.js"; +import type { DocEmbedder } from "./embed.js"; export type DocsVfsResult = | { kind: "ok"; body: string } | { kind: "not-found"; message: string }; +export interface DocsVfsOptions { + /** Query embedder (kind='query') for semantic `find/`. Absent → lexical only. */ + embedQuery?: DocEmbedder; +} + /** Resolve a `/docs/` subpath to rendered text from the docs table. */ export async function handleDocsVfs( subpath: string, query: QueryFn, tableName: string, + opts: DocsVfsOptions = {}, ): Promise { const path = subpath.replace(/^\/+/, "").replace(/\/+$/, ""); + // `find/` — hybrid semantic+lexical search over doc content. Checked + // BEFORE the directory/leaf resolution so `find` is never treated as a dir. + if (path === "find" || path.startsWith("find/")) { + const q = path === "find" ? "" : path.slice("find/".length).trim(); + if (q === "") { + return { kind: "ok", body: "Usage: cat /docs/find/ — search docs by meaning/keyword." }; + } + const queryEmbedding = opts.embedQuery ? await opts.embedQuery(q) : null; + const searchOpts: SearchOptions = { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: sqlLike(q), + queryEmbedding, + limit: 20, + }; + const hits = await searchDocs(query, tableName, searchOpts); + if (hits.length === 0) return { kind: "ok", body: `No docs match "${q}".` }; + const lines = [`${hits.length} doc(s) match "${q}"${queryEmbedding ? " (semantic + keyword)" : " (keyword)"}:`, ""]; + for (const h of hits) lines.push(`## ${h.path}\n${firstDocLine(h.content)}`); + lines.push("", `Open one with: cat /docs/.md`); + return { kind: "ok", body: lines.join("\n") }; + } + // Decide whether this is a directory index or a leaf doc. let dir: string | null = null; if (path === "" || path === "index.md") dir = ""; diff --git a/src/graph/session-context.ts b/src/graph/session-context.ts index 7ff1f331..61b62bec 100644 --- a/src/graph/session-context.ts +++ b/src/graph/session-context.ts @@ -154,6 +154,12 @@ export function graphContextLine(cwd: string, deps: GraphContextDeps = {}): stri " cat ~/.deeplake/memory/graph/neighborhood/ symbols + cross-file links", " Also: index.md · layers · tour · path//", "", + " PER-FILE DOCS (natural-language, kept fresh on commits — if generated):", + " cat ~/.deeplake/memory/docs/find/ semantic + keyword search over the", + " docs — use for \"where is X handled / how does Y work\"; returns ranked files.", + " cat ~/.deeplake/memory/docs/.md the doc for one source file.", + " Empty result just means no docs match (or none generated yet) — fall back to graph.", + "", " Then READ the files the graph points you to — don't answer from the graph", " alone. Cross-file calls/imports resolved for named imports across TS/JS/Python;", " bare (npm)/aliased/barrel/dynamic + instance-method dispatch stay unresolved.", diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index 24a78a7c..d00179eb 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -14,6 +14,7 @@ import { isDirectRun } from "../utils/direct-run.js"; import { type GrepParams, parseBashGrep, handleGrepDirect } from "./grep-direct.js"; import { handleGraphVfs } from "../graph/vfs-handler.js"; import { handleDocsVfs } from "../docs/vfs-handler.js"; +import { makeQueryEmbedder } from "../docs/embed.js"; import { executeCompiledBashCommand } from "./bash-command-compiler.js"; import { findVirtualPaths, @@ -456,7 +457,7 @@ export async function processPreToolUse(input: PreToolUseInput, deps: ClaudePreT if (virtualPath && (virtualPath === "/docs" || virtualPath.startsWith("/docs/")) && !virtualPath.endsWith("/")) { const subpath = virtualPath === "/docs" ? "" : virtualPath.slice("/docs/".length); logFn(`docs vfs: ${subpath || "(root)"}`); - const result = await handleDocsVfsFn(subpath, (sql) => api.query(sql), config.docsTableName); + const result = await handleDocsVfsFn(subpath, (sql) => api.query(sql), config.docsTableName, { embedQuery: makeQueryEmbedder() }); const body = result.kind === "ok" ? result.body : `(${result.kind}) ${result.message}`; if (input.tool_name === "Read") { const file_path = writeReadCacheFileFn(input.session_id, virtualPath, body); diff --git a/src/shell/grep-core.ts b/src/shell/grep-core.ts index cf7e116e..bb0c1bc6 100644 --- a/src/shell/grep-core.ts +++ b/src/shell/grep-core.ts @@ -426,6 +426,72 @@ export async function searchDeeplakeTables( })); } +/** + * Hybrid semantic+lexical search over the per-file docs table (`content` + + * `content_embedding`, `status='active'`). Mirrors {@link searchDeeplakeTables} + * for a single source, reusing the same primitives (float serialization, + * content filter, the `ARRAY_LENGTH(...) > 0` empty-vector guard, dedup-by-path) + * — so semantic scoring/ordering stays byte-identical to memory grep. Kept a + * SEPARATE entry point on purpose: docs must never leak into memory retrieval + * (see src/docs/vfs-handler.ts). `path` in the result is the doc's file id. + */ +export async function searchDocs( + query: (sql: string) => Promise[]>, + docsTable: string, + opts: SearchOptions, +): Promise { + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, multiWordPatterns } = opts; + const limit = opts.limit ?? 100; + const active = ` AND status = 'active'`; + const dedup = (rows: Record[]): ContentRow[] => { + const seen = new Set(); + const out: ContentRow[] = []; + for (const row of rows) { + const p = String(row["path"]); + if (seen.has(p)) continue; + seen.add(p); + out.push({ path: p, content: String(row["content"] ?? "") }); + } + return out; + }; + + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min(limit, Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20")); + const lexicalLimit = Math.min(limit, Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20")); + const filterPatternsForLex = contentScanOnly + ? (prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : (prefilterPattern ? [prefilterPattern] : [])) + : [escapedPattern]; + const lexFilter = buildContentFilter("content::text", likeOp, filterPatternsForLex); + const lexQuery = lexFilter + ? `SELECT doc_id AS path, content::text AS content, 1.0 AS score ` + + `FROM "${docsTable}" WHERE 1=1${pathFilter}${active}${lexFilter} LIMIT ${lexicalLimit}` + : null; + const semQuery = + `SELECT doc_id AS path, content::text AS content, (content_embedding <#> ${vecLit}) AS score ` + + `FROM "${docsTable}" WHERE ARRAY_LENGTH(content_embedding, 1) > 0${pathFilter}${active} ` + + `ORDER BY score DESC LIMIT ${semanticLimit}`; + const parts = [semQuery]; + if (lexQuery) parts.push(lexQuery); + const unionSql = parts.map(q => `(${q})`).join(" UNION ALL "); + const rows = await query( + `SELECT path, content, score FROM (${unionSql}) AS combined ORDER BY score DESC LIMIT ${semanticLimit + lexicalLimit}`, + ); + return dedup(rows); + } + + // Lexical-only (no query embedding available — daemon off / disabled). + const filterPatterns = contentScanOnly + ? (prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : (prefilterPattern ? [prefilterPattern] : [])) + : (multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]); + const filter = buildContentFilter("content::text", likeOp, filterPatterns); + const rows = await query( + `SELECT doc_id AS path, content::text AS content ` + + `FROM "${docsTable}" WHERE 1=1${pathFilter}${active}${filter} LIMIT ${limit}`, + ); + return dedup(rows); +} + function serializeFloat4Array(vec: number[]): string { const parts: string[] = []; for (const v of vec) { diff --git a/tests/claude-code/grep-core.test.ts b/tests/claude-code/grep-core.test.ts index 518204ef..be788ed8 100644 --- a/tests/claude-code/grep-core.test.ts +++ b/tests/claude-code/grep-core.test.ts @@ -10,8 +10,50 @@ import { refineGrepMatches, searchDeeplakeTables, grepBothTables, + searchDocs, } from "../../src/shell/grep-core.js"; +// ── searchDocs (per-file docs search — the docs/find backend) ───────────────── + +describe("searchDocs", () => { + const baseOpts = { pathFilter: "", contentScanOnly: false, likeOp: "ILIKE" as const, escapedPattern: "auth", limit: 20 }; + + it("hybrid: UNIONs a semantic (content_embedding <#>) + lexical query, active-only", async () => { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { calls.push(sql); return [{ path: "a.ts", content: "# A" }]; }); + const rows = await searchDocs(query, "hivemind_docs", { ...baseOpts, queryEmbedding: [0.1, 0.2, 0.3] }); + expect(calls).toHaveLength(1); + const sql = calls[0]; + expect(sql).toContain("content_embedding <#> "); // semantic branch + expect(sql).toContain("ARRAY_LENGTH(content_embedding, 1) > 0"); // empty-vector guard + expect(sql).toContain("status = 'active'"); // active only + expect(sql).toContain("UNION ALL"); // + lexical + expect(sql).toContain("ILIKE"); + expect(rows).toEqual([{ path: "a.ts", content: "# A" }]); + }); + + it("lexical-only when no query embedding: content ILIKE, no cosine operator", async () => { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { calls.push(sql); return []; }); + await searchDocs(query, "hivemind_docs", { ...baseOpts, queryEmbedding: null }); + expect(calls).toHaveLength(1); + expect(calls[0]).not.toContain("<#>"); + expect(calls[0]).toContain("status = 'active'"); + expect(calls[0]).toContain("ILIKE"); + }); + + it("dedups results by path (semantic winner kept over lexical dup)", async () => { + const query = vi.fn(async () => [ + { path: "dup.ts", content: "semantic" }, + { path: "dup.ts", content: "lexical" }, + { path: "b.ts", content: "b" }, + ]); + const rows = await searchDocs(query, "hivemind_docs", { ...baseOpts, queryEmbedding: [0.1] }); + expect(rows.map(r => r.path)).toEqual(["dup.ts", "b.ts"]); + expect(rows[0].content).toBe("semantic"); // first occurrence wins + }); +}); + // ── normalizeContent ──────────────────────────────────────────────────────── describe("normalizeContent: passthrough for non-session paths", () => { diff --git a/tests/shared/docs-backfill.test.ts b/tests/shared/docs-backfill.test.ts new file mode 100644 index 00000000..8a75eb6b --- /dev/null +++ b/tests/shared/docs-backfill.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it, vi } from "vitest"; +import { backfillDocEmbeddings } from "../../src/docs/backfill.js"; + +describe("backfillDocEmbeddings", () => { + it("embeds only docs missing a vector, one UPDATE each", async () => { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + if (sql.startsWith("SELECT")) return [ + { id: "1", content: "doc one", dims: null }, // missing → embed + { id: "2", content: "doc two", dims: 0 }, // empty → embed + { id: "3", content: "doc three", dims: 768 }, // has vector → skip + ]; + return []; + }); + const embed = vi.fn(async () => [0.1, 0.2, 0.3]); + const report = await backfillDocEmbeddings(query, "hivemind_docs", embed, 1); + expect(report).toEqual({ scanned: 3, embedded: 2, skipped: 1 }); + expect(embed).toHaveBeenCalledTimes(2); + const updates = calls.filter(c => c.startsWith("UPDATE")); + expect(updates).toHaveLength(2); + expect(updates[0]).toContain("content_embedding = "); + expect(updates.some(u => u.includes("WHERE id = '1'"))).toBe(true); + expect(updates.some(u => u.includes("WHERE id = '3'"))).toBe(false); // skipped + }); + + it("leaves the column NULL when the embedder is off (returns null)", async () => { + const calls: string[] = []; + const query = vi.fn(async (sql: string) => { + calls.push(sql); + return sql.startsWith("SELECT") ? [{ id: "1", content: "x", dims: null }] : []; + }); + const report = await backfillDocEmbeddings(query, "hivemind_docs", async () => null, 1); + expect(report.embedded).toBe(0); + expect(calls.filter(c => c.startsWith("UPDATE"))).toHaveLength(0); + }); +}); diff --git a/tests/shared/docs-vfs-handler.test.ts b/tests/shared/docs-vfs-handler.test.ts index 84e5b5f1..5bf29e85 100644 --- a/tests/shared/docs-vfs-handler.test.ts +++ b/tests/shared/docs-vfs-handler.test.ts @@ -80,3 +80,48 @@ describe("handleDocsVfs", () => { expect(r.kind === "not-found" && r.message).toContain("No such file"); }); }); + +describe("handleDocsVfs — find/ search route", () => { + const hitRows = [ + { path: "src/auth.ts", content: "# src/auth.ts\nHandles login and tokens." }, + { path: "src/session.ts", content: "# src/session.ts\nSession lifecycle." }, + ]; + + it("empty query → usage message, no SQL", async () => { + const query = vi.fn(async () => []); + const r = await handleDocsVfs("find", query, TBL); + expect(r.kind).toBe("ok"); + expect(r.kind === "ok" && r.body).toContain("Usage:"); + expect(query).not.toHaveBeenCalled(); + }); + + it("lexical (no embedder): lists ranked docs, marks (keyword)", async () => { + const query = vi.fn(async (_sql: string) => hitRows); + const r = await handleDocsVfs("find/login token", query, TBL); + expect(r.kind).toBe("ok"); + const body = r.kind === "ok" ? r.body : ""; + expect(body).toContain('2 doc(s) match "login token" (keyword)'); + expect(body).toContain("## src/auth.ts"); + expect(body).toContain("## src/session.ts"); + // searched with an ILIKE over content, no cosine (no embedding provided) + const sql = query.mock.calls[0][0] as string; + expect(sql).toContain("ILIKE"); + expect(sql).not.toContain("<#>"); + }); + + it("semantic (embedder returns a vector): marks (semantic + keyword) and runs cosine", async () => { + const query = vi.fn(async (_sql: string) => hitRows); + const embedQuery = vi.fn(async () => [0.1, 0.2, 0.3]); + const r = await handleDocsVfs("find/where are tokens minted", query, TBL, { embedQuery }); + expect(embedQuery).toHaveBeenCalledOnce(); + const body = r.kind === "ok" ? r.body : ""; + expect(body).toContain("(semantic + keyword)"); + expect((query.mock.calls[0][0] as string)).toContain("content_embedding <#>"); + }); + + it("no matches → friendly empty message", async () => { + const query = vi.fn(async () => []); + const r = await handleDocsVfs("find/nonexistent thing", query, TBL); + expect(r.kind === "ok" && r.body).toContain('No docs match "nonexistent thing"'); + }); +}); From 5bf81cb6357da60f429d57345024ad5aefc413e3 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 20:22:11 +0000 Subject: [PATCH 35/39] test: fix docs cli and hook tests for update-in-place and host seam --- tests/claude-code/cli-docs.test.ts | 19 ++++++++++++++----- .../claude-code/pre-tool-use-branches.test.ts | 6 +++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/claude-code/cli-docs.test.ts b/tests/claude-code/cli-docs.test.ts index 801a75c8..c8287cf2 100644 --- a/tests/claude-code/cli-docs.test.ts +++ b/tests/claude-code/cli-docs.test.ts @@ -33,7 +33,14 @@ vi.mock("../../src/graph/load-current.js", () => ({ loadCurrentSnapshot: (...a: unknown[]) => loadCurrentSnapshotMock(...a), })); vi.mock("../../src/docs/refresh-llm.js", () => ({ - makeClaudeGenerate: () => async () => "stub", + makeHostGenerate: () => async () => "stub", + makeHostGenerateDoc: () => async () => "stub", + makeHostBatchGenerateDoc: () => async () => new Map(), +})); +// Hermetic: no embed daemon round-trips in unit tests. +vi.mock("../../src/docs/embed.js", () => ({ + makeDocEmbedder: () => async () => null, + makeQueryEmbedder: () => async () => null, })); import { runDocsCommand } from "../../src/commands/docs.js"; @@ -163,7 +170,8 @@ describe("hivemind docs archive", () => { queryMock.mockResolvedValueOnce([docRow({ version: 2 })]); // getDocLatest await run(["archive", "a.ts"]); const sqls = queryMock.mock.calls.map((c) => c[0] as string); - expect(sqls.some((s) => /INSERT INTO "hivemind_docs"/.test(s) && /archived/.test(s))).toBe(true); + // UPDATE-in-place (F1): archive flips status on the existing row, not a new INSERT. + expect(sqls.some((s) => /UPDATE "hivemind_docs" SET/.test(s) && /status = 'archived'/.test(s))).toBe(true); expect(logged.join()).toMatch(/Archived doc a\.ts → v3/); }); }); @@ -211,9 +219,10 @@ describe("hivemind docs — anchored authoring + refresh over real files", () => queryMock.mockResolvedValue([docRow({ doc_id: "f.ts", content: "old body", anchors: stale })]); await run(["refresh", "--cwd", dir]); expect(logged.join("\n")).toMatch(/Refreshed 1/); - const insert = queryMock.mock.calls.map((c) => c[0] as string).find((s) => /INSERT INTO "hivemind_docs"/.test(s)); - expect(insert).toBeDefined(); - expect(insert!).toContain("stub"); // the generated body landed + // UPDATE-in-place (F1): the existing doc row is rewritten, not re-INSERTed. + const update = queryMock.mock.calls.map((c) => c[0] as string).find((s) => /UPDATE "hivemind_docs" SET/.test(s)); + expect(update).toBeDefined(); + expect(update!).toContain("stub"); // the generated body landed }); it("refresh prints a rejection outcome when the gate rejects (slow tier)", async () => { diff --git a/tests/claude-code/pre-tool-use-branches.test.ts b/tests/claude-code/pre-tool-use-branches.test.ts index 9b7d0ba7..a5327efd 100644 --- a/tests/claude-code/pre-tool-use-branches.test.ts +++ b/tests/claude-code/pre-tool-use-branches.test.ts @@ -214,7 +214,7 @@ describe("processPreToolUse: docs VFS routing", () => { { session_id: "s", tool_name: "Read", tool_input: { file_path: `${MEM_ABS}/docs/index.md` }, tool_use_id: "t" }, { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn }, ); - expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs"); + expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs", expect.anything()); // Read must get a file_path-shaped decision (cache file), not a real command. expect(d?.file_path).toContain("docs/index.md"); expect(d?.command).toBe(""); @@ -226,7 +226,7 @@ describe("processPreToolUse: docs VFS routing", () => { { session_id: "s", tool_name: "Read", tool_input: { file_path: `${MEM_ABS}/docs/src/graph/diff.ts.md` }, tool_use_id: "t" }, { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn }, ); - expect(handleDocsVfsFn).toHaveBeenCalledWith("src/graph/diff.ts.md", expect.any(Function), "hivemind_docs"); + expect(handleDocsVfsFn).toHaveBeenCalledWith("src/graph/diff.ts.md", expect.any(Function), "hivemind_docs", expect.anything()); }); it("routes a Bash cat of a docs path to a command-shaped allow decision", async () => { @@ -235,7 +235,7 @@ describe("processPreToolUse: docs VFS routing", () => { { session_id: "s", tool_name: "Bash", tool_input: { command: "cat ~/.deeplake/memory/docs/index.md" }, tool_use_id: "t" }, { config: DOCS_CONFIG, createApi: () => makeApi(), handleDocsVfsFn, logFn: vi.fn() }, ); - expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs"); + expect(handleDocsVfsFn).toHaveBeenCalledWith("index.md", expect.any(Function), "hivemind_docs", expect.anything()); expect(d?.command).toContain("Docs Index"); }); }); From d2c9f2305c38c21aadb31d858b75fb150a7eb4d2 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 20:52:57 +0000 Subject: [PATCH 36/39] feat: expose docs search as an mcp tool for cross-agent use --- src/mcp/server.ts | 48 +++++++++++++++++++++++++--- tests/claude-code/mcp-server.test.ts | 4 +-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 3bcd4c80..369be934 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -2,9 +2,10 @@ * Hivemind MCP server — exposes shared org memory as MCP tools. * * Tools: - * hivemind_search — keyword/regex search across summaries + sessions - * hivemind_read — read full content of a specific memory path - * hivemind_index — list summaries with their dates and descriptions + * hivemind_search — keyword/regex search across summaries + sessions + * hivemind_docs_search — hybrid semantic/lexical search over per-file code docs + * hivemind_read — read full content of a specific memory path + * hivemind_index — list summaries with their dates and descriptions * * Transport: stdio. Spawned as a subprocess by the consuming MCP client * (Hermes today; reused by any future MCP-aware agent). @@ -21,13 +22,15 @@ import { loadConfig } from "../config.js"; import { DeeplakeApi } from "../deeplake-api.js"; import { isMissingTableError } from "../deeplake-schema.js"; import { sqlStr, sqlLike } from "../utils/sql.js"; -import { searchDeeplakeTables, buildGrepSearchOptions, normalizeContent, TRUNCATION_NOTICE, type GrepMatchParams } from "../shell/grep-core.js"; +import { searchDeeplakeTables, searchDocs, buildGrepSearchOptions, normalizeContent, TRUNCATION_NOTICE, type GrepMatchParams } from "../shell/grep-core.js"; +import { makeQueryEmbedder } from "../docs/embed.js"; import { getVersion } from "../cli/version.js"; interface ServerContext { api: DeeplakeApi; memoryTable: string; sessionsTable: string; + docsTable: string; } function getContext(): ServerContext | { error: string } { @@ -40,7 +43,7 @@ function getContext(): ServerContext | { error: string } { return { error: "Hivemind config could not be loaded — credentials present but invalid." }; } const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName); - return { api, memoryTable: config.tableName, sessionsTable: config.sessionsTableName }; + return { api, memoryTable: config.tableName, sessionsTable: config.sessionsTableName, docsTable: config.docsTableName }; } function errorResult(text: string): { content: Array<{ type: "text"; text: string }> } { @@ -108,6 +111,41 @@ server.registerTool( }, ); +server.registerTool( + "hivemind_docs_search", + { + description: "Search the per-file CODE documentation (kept fresh on commits) by meaning or keyword. Hybrid semantic + lexical. Use for 'where is X handled / how does Y work / which file does Z' about the current codebase — returns the most relevant source files with a one-line summary. Different from hivemind_search (that's past sessions/conversations; this is code docs).", + inputSchema: { + query: z.string().describe("Natural-language question or keywords about the codebase."), + limit: z.number().int().min(1).max(50).optional().describe("Maximum docs to return (default 10)."), + }, + }, + async ({ query, limit }: { query: string; limit?: number }) => { + const ctx = getContext(); + if ("error" in ctx) return errorResult(ctx.error); + + const params: GrepMatchParams = { + pattern: query, ignoreCase: true, wordMatch: false, filesOnly: false, + countOnly: false, lineNumber: false, invertMatch: false, fixedString: true, + }; + const opts = buildGrepSearchOptions(params, "/"); + opts.limit = limit ?? 10; + // Same rail as memory search: semantic when embeddings are on, else lexical. + opts.queryEmbedding = await makeQueryEmbedder()(query); + + try { + const rows = await searchDocs((sql) => ctx.api.query(sql), ctx.docsTable, opts); + if (rows.length === 0) return errorResult(`No docs match "${query}".`); + const lines = rows.map(r => `[${r.path}]\n${r.content.slice(0, 600)}`); + return { content: [{ type: "text", text: lines.join("\n\n---\n\n") }] }; + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + if (isMissingTableError(msg)) return errorResult(`No docs match "${query}". ${FRESH_ORG_HINT}`); + return errorResult(`Docs search failed: ${msg}`); + } + }, +); + server.registerTool( "hivemind_read", { diff --git a/tests/claude-code/mcp-server.test.ts b/tests/claude-code/mcp-server.test.ts index 2b9c099e..56f1086a 100644 --- a/tests/claude-code/mcp-server.test.ts +++ b/tests/claude-code/mcp-server.test.ts @@ -87,10 +87,10 @@ beforeEach(() => { afterEach(() => { vi.restoreAllMocks(); }); describe("MCP server — registration shape", () => { - it("registers exactly the three hivemind tools, named and described", async () => { + it("registers exactly the hivemind tools, named and described", async () => { await importServer(); expect(Array.from(registeredTools.keys()).sort()).toEqual([ - "hivemind_index", "hivemind_read", "hivemind_search", + "hivemind_docs_search", "hivemind_index", "hivemind_read", "hivemind_search", ]); for (const tool of registeredTools.values()) { expect(typeof tool.config.description).toBe("string"); From 0f38eb5c7df4f29ae53abc6a8c6e64060f3763f4 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 21:14:32 +0000 Subject: [PATCH 37/39] feat: serve the docs VFS from the deeplake shell (browse + find) --- src/shell/deeplake-fs.ts | 48 ++++++++++++++++++++++++++- src/shell/deeplake-shell.ts | 3 +- tests/claude-code/deeplake-fs.test.ts | 37 +++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/shell/deeplake-fs.ts b/src/shell/deeplake-fs.ts index 9de39d47..4637fe19 100644 --- a/src/shell/deeplake-fs.ts +++ b/src/shell/deeplake-fs.ts @@ -21,6 +21,8 @@ import { type PathKind, } from "./goal-paths.js"; import { handleGraphVfs } from "../graph/vfs-handler.js"; +import { handleDocsVfs } from "../docs/vfs-handler.js"; +import { makeQueryEmbedder } from "../docs/embed.js"; interface ReadFileOptions { encoding?: BufferEncoding } interface WriteFileOptions { encoding?: BufferEncoding } @@ -150,6 +152,25 @@ function readGraphFile(p: string, cwd: string): string { throw fsErr("ENOENT", `${r.message}`, p); } +// Per-file docs live at /docs/. Same thin-bridge pattern as +// /graph/: detect the prefix, strip it, delegate to handleDocsVfs (the same +// dispatcher the pre-tool-use hook uses). Unlike graph, docs are backed by a +// SQL table so the bridge is ASYNC and needs the query seam + docs table. +const DOCS_ROOT = "/docs"; +const DOCS_PREFIX = "/docs/"; +const DOCS_DIRS = new Set([DOCS_ROOT, "/docs/find"]); + +function isDocsPath(p: string): boolean { + return p === DOCS_ROOT || p.startsWith(DOCS_PREFIX); +} +function isDocsDir(p: string): boolean { + return DOCS_DIRS.has(p); +} +function docsSubpathOf(p: string): string { + if (p === DOCS_ROOT) return ""; + return p.slice(DOCS_PREFIX.length); +} + export class DeeplakeFs implements IFileSystem { // path → Buffer (content) or null (exists but not fetched yet) private files = new Map(); @@ -177,6 +198,8 @@ export class DeeplakeFs implements IFileSystem { // the goal/kpi routing is disabled (test or legacy configurations). private goalsTable: string | null = null; private kpisTable: string | null = null; + // Per-file docs table, for /docs/ VFS reads + docs/find search. Null = off. + private docsTable: string | null = null; // Embedding client lazily created on first flush. Lives as long as the process. private embedClient: EmbedClient | null = null; @@ -195,12 +218,13 @@ export class DeeplakeFs implements IFileSystem { table: string, mount = "/memory", sessionsTable?: string, - extra?: { goalsTable?: string; kpisTable?: string }, + extra?: { goalsTable?: string; kpisTable?: string; docsTable?: string }, ): Promise { const fs = new DeeplakeFs(client, table, mount); fs.sessionsTable = sessionsTable ?? null; fs.goalsTable = extra?.goalsTable ?? null; fs.kpisTable = extra?.kpisTable ?? null; + fs.docsTable = extra?.docsTable ?? null; // Ensure the memory table + goal/kpi tables exist before // bootstrapping. Each ensure call is idempotent and lazy-heals // any column drift from prior schema versions. Failures bubble @@ -712,6 +736,15 @@ export class DeeplakeFs implements IFileSystem { if (isGraphDir(p)) throw fsErr("EISDIR", "illegal operation on a directory", p); return readGraphFile(p, process.cwd()); } + // Docs VFS bridge — same as graph, but async (SQL-backed) and delegating to + // handleDocsVfs. Makes browse + docs/find work in the shell path (codex / + // cursor / hermes / interactive), not just the pre-tool-use hook (Claude). + if (isDocsPath(p) && this.docsTable) { + if (isDocsDir(p)) throw fsErr("EISDIR", "illegal operation on a directory", p); + const r = await handleDocsVfs(docsSubpathOf(p), (sql) => this.client.query(sql), this.docsTable, { embedQuery: makeQueryEmbedder() }); + if (r.kind === "ok") return r.body; + throw fsErr("ENOENT", `${r.message}`, p); + } if (this.dirs.has(p) && !this.files.has(p)) throw fsErr("EISDIR", "illegal operation on a directory", p); // Virtual index.md: if no real row exists, generate from summary rows @@ -860,6 +893,9 @@ export class DeeplakeFs implements IFileSystem { const r = handleGraphVfs(graphSubpathOf(p), process.cwd()); return r.kind === "ok" || r.kind === "no-graph"; } + // Docs VFS — dirs always exist; a leaf/find query is addressable (readFile + // renders content or a friendly "no match" body, never not-found for find). + if (isDocsPath(p) && this.docsTable) return true; return this.files.has(p) || this.dirs.has(p); } @@ -885,6 +921,16 @@ export class DeeplakeFs implements IFileSystem { mtime: new Date(), }; } + // Docs VFS — /docs and /docs/find are directories; any other /docs/ path is + // a (synthesized) file. Kept cheap (no SQL); readFile surfaces a real + // not-found as ENOENT. + if (isDocsPath(p) && this.docsTable) { + const dir = isDocsDir(p); + return { + isFile: !dir, isDirectory: dir, isSymbolicLink: false, + mode: dir ? 0o755 : 0o644, size: 0, mtime: new Date(), + }; + } const isFile = this.files.has(p); const isDir = this.dirs.has(p); // Virtual index.md: always exists as a file diff --git a/src/shell/deeplake-shell.ts b/src/shell/deeplake-shell.ts index f99df7b1..b53a9259 100644 --- a/src/shell/deeplake-shell.ts +++ b/src/shell/deeplake-shell.ts @@ -54,6 +54,7 @@ async function main(): Promise { const sessionsTable = process.env["HIVEMIND_SESSIONS_TABLE"] ?? "sessions"; const goalsTable = process.env["HIVEMIND_GOALS_TABLE"] ?? config.goalsTableName; const kpisTable = process.env["HIVEMIND_KPIS_TABLE"] ?? config.kpisTableName; + const docsTable = process.env["HIVEMIND_DOCS_TABLE"] ?? config.docsTableName; const mount = process.env["HIVEMIND_MOUNT"] ?? "/"; const client = new DeeplakeApi( @@ -64,7 +65,7 @@ async function main(): Promise { process.stderr.write(`Connecting to deeplake://${config.workspaceId}/${table} ...\n`); } - const fs = await DeeplakeFs.create(client, table, mount, sessionsTable, { goalsTable, kpisTable }); + const fs = await DeeplakeFs.create(client, table, mount, sessionsTable, { goalsTable, kpisTable, docsTable }); if (!isOneShot) { const fileCount = fs.getAllPaths().filter(p => !!p).length; diff --git a/tests/claude-code/deeplake-fs.test.ts b/tests/claude-code/deeplake-fs.test.ts index 52a60e68..874f4139 100644 --- a/tests/claude-code/deeplake-fs.test.ts +++ b/tests/claude-code/deeplake-fs.test.ts @@ -1,4 +1,9 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +// Hermetic: no embed daemon round-trips from the /docs/find query embedder. +vi.mock("../../src/docs/embed.js", () => ({ + makeDocEmbedder: () => async () => null, + makeQueryEmbedder: () => async () => null, +})); import { DeeplakeFs, guessMime } from "../../src/shell/deeplake-fs.js"; // ── Mock ManagedClient ──────────────────────────────────────────────────────── @@ -1140,3 +1145,35 @@ describe("cp recursive", () => { expect(await fs.readFile("/memory/src/a.txt")).toBe("aaa"); }); }); + +// ── docs VFS bridge (the shell path used by codex/cursor/hermes/interactive) ── + +describe("docs VFS routing in the shell", () => { + function makeDocsClient(onQuery: (sql: string) => unknown[]) { + return { + query: vi.fn(async (sql: string) => onQuery(sql)), + ensureTable: async () => {}, + ensureGoalsTable: async () => {}, ensureKpisTable: async () => {}, + }; + } + + it("routes cat /docs/find/ to the DOCS table (not the memory read)", async () => { + const calls: string[] = []; + const client = makeDocsClient((sql) => { + calls.push(sql); + if (/FROM "hivemind_docs"/.test(sql)) return [{ path: "src/utils/sql.ts", content: "# sql.ts\nescaping helpers" }]; + return []; + }); + const fs = await DeeplakeFs.create(client as never, "memory", "/", "sessions", { docsTable: "hivemind_docs" }); + const out = await fs.readFile("/docs/find/injection"); + expect(out).toContain("src/utils/sql.ts"); // rendered the doc hit + expect(calls.some((s) => /FROM "hivemind_docs"/.test(s))).toBe(true); // queried docs table + }); + + it("does NOT route /docs when docsTable is unset (feature off)", async () => { + const client = makeDocsClient(() => []); + const fs = await DeeplakeFs.create(client as never, "memory", "/"); + // No docsTable → /docs/find falls through to the generic memory read → ENOENT. + await expect(fs.readFile("/docs/find/x")).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); From 2cc19070fecc3f703bc8a8b4bab40adca2ec74cf Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 21:14:37 +0000 Subject: [PATCH 38/39] feat: intercept docs VFS in the codex pre-tool-use hook --- src/hooks/codex/pre-tool-use.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/hooks/codex/pre-tool-use.ts b/src/hooks/codex/pre-tool-use.ts index 49fda771..824200c6 100644 --- a/src/hooks/codex/pre-tool-use.ts +++ b/src/hooks/codex/pre-tool-use.ts @@ -22,6 +22,8 @@ import { DeeplakeApi } from "../../deeplake-api.js"; import { sqlLike } from "../../utils/sql.js"; import { parseBashGrep, handleGrepDirect } from "../grep-direct.js"; import { tryGraphRead } from "../../graph/graph-command.js"; +import { handleDocsVfs } from "../../docs/vfs-handler.js"; +import { makeQueryEmbedder } from "../../docs/embed.js"; import { executeCompiledBashCommand } from "../bash-command-compiler.js"; import { findVirtualPaths, @@ -239,6 +241,19 @@ export async function processCodexPreToolUse( } } + // Docs VFS dispatch — a cat of `/docs/*` (browse or find/) is answered by + // the docs table via handleDocsVfs, NOT the generic memory read below + // (docs live in their own table). Mirrors the graph dispatch above; async + // + config-backed. Same route Claude's pre-tool-use uses. + if (virtualPath && (virtualPath === "/docs" || virtualPath.startsWith("/docs/"))) { + logFn(`docs vfs intercept: ${virtualPath}`); + const docsTable = process.env["HIVEMIND_DOCS_TABLE"] ?? config.docsTableName; + const sub = virtualPath === "/docs" ? "" : virtualPath.slice("/docs/".length); + const r = await handleDocsVfs(sub, (sql) => api.query(sql), docsTable, { embedQuery: makeQueryEmbedder() }); + const body = r.kind === "ok" ? r.body : `${virtualPath}: No such file or directory`; + return { action: "block", output: body, rewrittenCommand: rewritten }; + } + if (virtualPath && !virtualPath.endsWith("/")) { logFn(`direct read: ${virtualPath}`); let content = virtualPath === "/index.md" From a0889cfef9037b946219b8383997b0dc75c3c8e4 Mon Sep 17 00:00:00 2001 From: Emanuele Fenocchi Date: Thu, 2 Jul 2026 22:12:48 +0000 Subject: [PATCH 39/39] feat: intercept docs VFS in the cursor hook via shared tryDocsRead --- src/docs/docs-command.ts | 48 +++++++++++++++++++++++++++++++ src/graph/graph-command.ts | 4 +-- src/hooks/cursor/pre-tool-use.ts | 24 ++++++++++++++-- tests/shared/docs-command.test.ts | 25 ++++++++++++++++ 4 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/docs/docs-command.ts create mode 100644 tests/shared/docs-command.test.ts diff --git a/src/docs/docs-command.ts b/src/docs/docs-command.ts new file mode 100644 index 00000000..5536f3b0 --- /dev/null +++ b/src/docs/docs-command.ts @@ -0,0 +1,48 @@ +/** + * Shared `cat /docs/*` dispatcher for the rewrite-capable agent hooks + * (claude / codex / cursor / hermes). The docs analogue of + * `tryGraphRead` (src/graph/graph-command.ts): parse a read command, and if it + * targets the `/docs/` subtree, answer it from the docs table via handleDocsVfs. + * + * Reuses the graph command parser (`parseReadTargetPath`, `stripQuotes`, + * `hasTraversal`) so every agent routes docs the exact same way it routes the + * graph and sessions — no per-agent reinvention. Async because docs are + * SQL-backed (graph reads a local snapshot synchronously). + */ + +import { handleDocsVfs, type DocsVfsOptions } from "./vfs-handler.js"; +import type { QueryFn } from "./read.js"; +import { parseReadTargetPath, stripQuotes, hasTraversal } from "../graph/graph-command.js"; + +const DOCS_ROOT = "/docs"; +const DOCS_PREFIX = "/docs/"; +const DOCS_LISTING = "index.md\nfind/\n"; + +/** + * If `rewrittenCommand` is a read (cat/head/tail/ls) targeting `/docs/*`, return + * the rendered body; otherwise null (caller falls through to its normal read). + */ +export async function tryDocsRead( + rewrittenCommand: string, + query: QueryFn, + docsTable: string, + opts: DocsVfsOptions = {}, +): Promise { + // `ls /docs` (+ trailing-slash/flag variants) → directory listing. + const ls = rewrittenCommand.replace(/\s+2>\S+/g, "").trim().match(/^ls\s+(?:-\S+\s+)*(\S+)\s*$/); + if (ls) { + const dir = stripQuotes(ls[1]!).replace(/\/+$/, "") || "/"; + return dir === DOCS_ROOT ? DOCS_LISTING : null; + } + + const virtualPath = parseReadTargetPath(rewrittenCommand); + if (virtualPath === null) return null; + if (hasTraversal(virtualPath)) return null; // /docs/../secret escapes the subtree + + const normalized = virtualPath.replace(/\/+$/, "") || "/"; + if (normalized !== DOCS_ROOT && !virtualPath.startsWith(DOCS_PREFIX)) return null; + + const sub = normalized === DOCS_ROOT ? "" : virtualPath.slice(DOCS_PREFIX.length); + const r = await handleDocsVfs(sub, query, docsTable, opts); + return r.kind === "ok" ? r.body : `(${r.kind}) ${r.message}`; +} diff --git a/src/graph/graph-command.ts b/src/graph/graph-command.ts index 78c33349..67745a90 100644 --- a/src/graph/graph-command.ts +++ b/src/graph/graph-command.ts @@ -25,7 +25,7 @@ function tokenize(s: string): string[] { } /** Strip one layer of surrounding matching quotes from a token. */ -function stripQuotes(p: string): string { +export function stripQuotes(p: string): string { if (p.length >= 2 && ((p[0] === '"' && p[p.length - 1] === '"') || (p[0] === "'" && p[p.length - 1] === "'"))) { return p.slice(1, -1); } @@ -86,7 +86,7 @@ export function parseReadTargetPath(rewrittenCommand: string): string | null { } /** True when a virtual path tries to escape the graph subtree via a `..` segment. */ -function hasTraversal(virtualPath: string): boolean { +export function hasTraversal(virtualPath: string): boolean { return virtualPath.split("/").includes(".."); } diff --git a/src/hooks/cursor/pre-tool-use.ts b/src/hooks/cursor/pre-tool-use.ts index 2fc5eeca..d806121e 100644 --- a/src/hooks/cursor/pre-tool-use.ts +++ b/src/hooks/cursor/pre-tool-use.ts @@ -33,6 +33,8 @@ import { log as _log } from "../../utils/debug.js"; import { parseBashGrep, handleGrepDirect } from "../grep-direct.js"; import { touchesMemory, rewritePaths } from "../memory-path-utils.js"; import { tryGraphRead } from "../../graph/graph-command.js"; +import { tryDocsRead } from "../../docs/docs-command.js"; +import { makeQueryEmbedder } from "../../docs/embed.js"; const log = (msg: string) => _log("cursor-pre-tool-use", msg); interface CursorShellToolInput { @@ -79,9 +81,6 @@ async function main(): Promise { return; } - const grepParams = parseBashGrep(rewritten); - if (!grepParams) return; // not a grep/rg invocation we can handle directly - const config = loadConfig(); if (!config) { log("no config — falling through to Cursor's bash"); @@ -96,6 +95,25 @@ async function main(): Promise { config.tableName, ); + // Docs VFS dispatch — a cat of /docs/* (browse or find/) answered from the + // docs table, same rewrite trick as the graph dispatch above. Runs before the + // grep parse so `cat /docs/find/x` (not a grep) isn't left to Cursor's host bash. + const docsTable = process.env["HIVEMIND_DOCS_TABLE"] ?? config.docsTableName; + const docsBody = await tryDocsRead(rewritten, (sql) => api.query(sql), docsTable, { embedQuery: makeQueryEmbedder() }); + if (docsBody !== null) { + log(`docs vfs intercept: ${command.slice(0, 80)}`); + const echoCmd = `cat <<'__HIVEMIND_RESULT__'\n${docsBody}\n__HIVEMIND_RESULT__`; + process.stdout.write(JSON.stringify({ + permission: "allow", + updated_input: { command: echoCmd }, + agent_message: "[Hivemind docs]", + })); + return; + } + + const grepParams = parseBashGrep(rewritten); + if (!grepParams) return; // not a grep/rg invocation we can handle directly + try { const result = await handleGrepDirect(api, config.tableName, config.sessionsTableName, grepParams); if (result === null) { diff --git a/tests/shared/docs-command.test.ts b/tests/shared/docs-command.test.ts new file mode 100644 index 00000000..e3f539d4 --- /dev/null +++ b/tests/shared/docs-command.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it, vi } from "vitest"; +vi.mock("../../src/docs/embed.js", () => ({ makeDocEmbedder: () => async () => null, makeQueryEmbedder: () => async () => null })); +import { tryDocsRead } from "../../src/docs/docs-command.js"; + +// query mock: docs-table SELECT returns one hit; anything else empty. +const query = async (sql: string) => (/FROM "docs"/.test(sql) ? [{ path: "src/utils/sql.ts", content: "# sql.ts\nescaping" }] : []); + +describe("tryDocsRead", () => { + it("dispatches cat /docs/find/ to the docs table and renders hits", async () => { + const out = await tryDocsRead(`cat /docs/find/injection`, query, "docs"); + expect(out).toContain("src/utils/sql.ts"); + }); + it("ls /docs → directory listing (index.md + find/)", async () => { + const out = await tryDocsRead(`ls /docs`, query, "docs"); + expect(out).toContain("find/"); + expect(out).toContain("index.md"); + }); + it("returns null for a non-/docs read (caller falls through)", async () => { + expect(await tryDocsRead(`cat /summaries/alice/x.md`, query, "docs")).toBeNull(); + expect(await tryDocsRead(`grep foo /`, query, "docs")).toBeNull(); + }); + it("refuses path traversal out of /docs", async () => { + expect(await tryDocsRead(`cat /docs/../secret`, query, "docs")).toBeNull(); + }); +});