Rebuild Symbol HNSW around incremental embedding writes

Author: GlitterKill

CHANGELOG.md

@@ -12,10 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - **CPU embedding auto-tuning**: Omitted local CPU settings now use physical-core and startup free-memory bounds for embedding concurrency, symbol batch 8, and the physical-core width for automatic ONNX intra-op threads. Auto-detected and pinned performance tiers share these presets, while explicit settings remain authoritative; remove explicit `embeddingConcurrency`, `embeddingBatchSize`, or positive `onnx.intraOpNumThreads` fields to adopt the automatic values.
-- **Incremental symbol embedding refresh**: Only changed `SymbolVectorEmbedding` rows are deleted and reinserted while a healthy HNSW is retained; the index is created only for bootstrap or missing-index recovery after at least one complete model row exists. Repository-local cleanup removes only changed-file ownership links, preserves shared Symbol vectors, and deterministically transfers their canonical owner. Symbol ANN queries filter through repository-scoped projected graphs before ranking. In a direct spike, this was 18.5x faster than the prior drop-and-rebuild path for 50 changed rows out of 26,000, with ANN results durable after close/reopen; this is evidence, not an SLA. Migration 26 copies complete legacy vectors in-database. Legacy `Symbol` embedding columns remain inert for compatibility, and upgraded databases may also retain legacy `Symbol` HNSW indexes until a safe rebuild.
+- **Incremental symbol embedding refresh**: Only changed `SymbolVectorEmbedding` rows are deleted and reinserted. When the shared table already has a model-specific HNSW, refresh checkpoints, drops the index once, replaces all changed rows, recreates the index once, and checkpoints again; bootstrap or missing-index recovery creates it after at least one complete model row exists. Repository-local cleanup removes only changed-file ownership links, preserves shared Symbol vectors, and deterministically transfers their canonical owner. Symbol ANN queries filter through repository-scoped projected graphs before ranking. Migration 26 copies complete legacy vectors in-database. Legacy `Symbol` embedding columns remain inert for compatibility, and upgraded databases may also retain legacy `Symbol` HNSW indexes until a safe rebuild.
 
 ### Fixed
 
+- **Multi-repository Jina indexing crash**: Prevent later repositories from writing new `SymbolVectorEmbedding` rows through a large live LadybugDB HNSW, which could terminate the process after earlier repositories had populated the shared index.
+
 ## [0.13.5] - 2026-08-25
 
 ### Added

docs/architecture.md

@@ -305,7 +305,7 @@ Read pool enables concurrent multi-session reads (4-6 MCP sessions). Write seria
 | **SummaryCache**    | symbolId, summary, provider, model, cardHash, costUsd |
 | **SymbolReference** | referenceId, symbolId, file, line                     |
 
-Production Symbol vectors live in `SymbolVectorEmbedding`, with one complete row per Symbol and model. Model-specific HNSW indexes target this table. Incremental indexing and background semantic repair delete and replace only changed embedding rows while retaining a healthy HNSW; index bootstrap occurs only when the exact table/name/type/property identity is absent and at least one complete row exists. Symbol ANN queries rank candidates inside a repository-scoped projected graph, so matching Symbol-to-File-to-Repo ownership filters the graph before the top-K search.
+Production Symbol vectors live in `SymbolVectorEmbedding`, with one complete row per Symbol and model. Model-specific HNSW indexes target this table. Incremental indexing and background semantic repair delete and replace only changed embedding rows inside one checkpointed HNSW drop/write/recreate cycle; index bootstrap occurs only when the exact table/name/type/property identity is absent and at least one complete row exists. Symbol ANN queries rank candidates inside a repository-scoped projected graph, so matching Symbol-to-File-to-Repo ownership filters the graph before the top-K search.
 
 **Sync, policy, and memory nodes:**
 

src/indexer/embeddings.ts

@@ -31,6 +31,7 @@ import {
 } from "../db/ladybug-symbol-embeddings.js";
 import {
   createVectorIndex,
+  dropVectorIndex,
   showIndexesStrict,
 } from "../retrieval/index-lifecycle.js";
 import {
@@ -422,63 +423,59 @@ export async function refreshSymbolEmbeddings(params: {
     shouldBootstrapIndex = configuredIndex === undefined;
   }
 
-  const bootstrapVectorIndex = async (): Promise<void> => {
-    if (
-      !shouldBootstrapIndex ||
-      vecProp === null ||
-      indexName === null ||
-      !modelInfo
-    ) {
-      return;
+  const createRequiredVectorIndex = async (): Promise<void> => {
+    if (vecProp === null || indexName === null || !modelInfo) return;
+    recordMemorySnapshot("beforeHnsw");
+    params.onProgress?.({
+      stage: "embeddings",
+      substage: "symbolVectorIndex",
+      current: Math.min(skipped + embedded, symbols.length),
+      total: symbols.length,
+      model: storageModel,
+      message: "building HNSW",
+    });
+    let ok: boolean;
+    try {
+      ok = await measure("hnsw.create", () =>
+        withWriteConn((wConn) =>
+          createVectorIndex(
+            wConn,
+            SYMBOL_VECTOR_EMBEDDING_TABLE,
+            vecProp,
+            indexName,
+            modelInfo.dimension,
+            params.vectorEfc,
+          ),
+        ),
+      );
+    } finally {
+      recordMemorySnapshot("afterHnsw");
+    }
+    params.onProgress?.({
+      stage: "embeddings",
+      substage: "symbolVectorIndex",
+      current: Math.min(skipped + embedded, symbols.length),
+      total: symbols.length,
+      model: storageModel,
+      message: ok ? "ready" : "creation failed",
+    });
+    if (!ok) {
+      throw new IndexError(
+        `Failed to create required vector index '${indexName}' on ${SYMBOL_VECTOR_EMBEDDING_TABLE}.${vecProp}`,
+      );
     }
+    logger.info(
+      `[embeddings] Vector index '${indexName}' created on ${SYMBOL_VECTOR_EMBEDDING_TABLE}`,
+    );
+  };
+
+  const bootstrapVectorIndex = async (): Promise<void> => {
+    if (!shouldBootstrapIndex) return;
     if (!(await hasCompleteSymbolVectorEmbedding(conn, modelName))) return;
     await runHnswRebuildCycle(
       "symbol-vector-bootstrap-pre-create",
       "symbol-vector-bootstrap-post-create",
-      async () => {
-        recordMemorySnapshot("beforeHnsw");
-        params.onProgress?.({
-          stage: "embeddings",
-          substage: "symbolVectorIndex",
-          current: Math.min(skipped + embedded, symbols.length),
-          total: symbols.length,
-          model: storageModel,
-          message: "building HNSW",
-        });
-        let ok: boolean;
-        try {
-          ok = await measure("hnsw.create", () =>
-            withWriteConn((wConn) =>
-              createVectorIndex(
-                wConn,
-                SYMBOL_VECTOR_EMBEDDING_TABLE,
-                vecProp,
-                indexName,
-                modelInfo.dimension,
-                params.vectorEfc,
-              ),
-            ),
-          );
-        } finally {
-          recordMemorySnapshot("afterHnsw");
-        }
-        params.onProgress?.({
-          stage: "embeddings",
-          substage: "symbolVectorIndex",
-          current: Math.min(skipped + embedded, symbols.length),
-          total: symbols.length,
-          model: storageModel,
-          message: ok ? "ready" : "creation failed",
-        });
-        if (!ok) {
-          throw new IndexError(
-            `Failed to create required vector index '${indexName}' on ${SYMBOL_VECTOR_EMBEDDING_TABLE}.${vecProp}`,
-          );
-        }
-        logger.info(
-          `[embeddings] Vector index '${indexName}' created on ${SYMBOL_VECTOR_EMBEDDING_TABLE}`,
-        );
-      },
+      createRequiredVectorIndex,
       params.postIndexSessionTimeoutMs,
       params.recordTiming,
       params.repoId,
@@ -542,6 +539,16 @@ export async function refreshSymbolEmbeddings(params: {
   // throughput optimisation.
   uncachedItems.sort((a, b) => a.prefixedText.length - b.prefixedText.length);
 
+  // LadybugDB can terminate the process when a large live HNSW receives new
+  // vector rows. Keep every changed Symbol write inside one drop/recreate
+  // cycle; an absent bootstrap index remains on the cheaper create-only path.
+  const useRebuildPath =
+    !shouldBootstrapIndex &&
+    vecProp !== null &&
+    indexName !== null &&
+    modelInfo !== undefined &&
+    uncachedItems.length > 0;
+
   // Resolve concurrency: clamp to [1, MAX_EMBEDDING_CONCURRENCY].
   const maxConcurrency = Math.max(
     1,
@@ -757,7 +764,7 @@ export async function refreshSymbolEmbeddings(params: {
       }
       recordMemorySnapshot("afterInference");
     } finally {
-      await bootstrapVectorIndex();
+      if (!useRebuildPath) await bootstrapVectorIndex();
     }
 
     // Progress: fire at end through fireProgress() so the monotonic clamp
@@ -771,5 +778,36 @@ export async function refreshSymbolEmbeddings(params: {
       ? { embedded, skipped, degraded: true }
       : { embedded, skipped };
   };
-  return runPersistenceCycle();
+  if (!useRebuildPath || indexName === null) return runPersistenceCycle();
+  return runHnswRebuildCycle(
+    "symbol-vector-rebuild-pre-drop",
+    "symbol-vector-rebuild-post-create",
+    async () => {
+      const dropResult = await measure("hnsw.drop", () =>
+        withWriteConn((wConn) =>
+          dropVectorIndex(
+            wConn,
+            SYMBOL_VECTOR_EMBEDDING_TABLE,
+            indexName,
+          ),
+        ),
+      );
+      if (dropResult.status === "failed") {
+        throw new IndexError(
+          `Failed to drop required vector index '${indexName}' before Symbol embedding writes: ${dropResult.error}`,
+        );
+      }
+      logger.info(
+        `[embeddings] Dropped vector index '${indexName}' for ${uncachedItems.length} changed Symbol vector(s)`,
+      );
+      try {
+        return await runPersistenceCycle();
+      } finally {
+        await createRequiredVectorIndex();
+      }
+    },
+    params.postIndexSessionTimeoutMs,
+    params.recordTiming,
+    params.repoId,
+  );
 }

tests/integration/semantic-embedding.test.ts

@@ -891,7 +891,7 @@ describe("Semantic Embedding Pipeline", () => {
     );
   });
 
-  it("bootstraps HNSW once and retains it during incremental refresh", async () => {
+  it("bootstraps HNSW and rebuilds it during incremental refresh", async () => {
     const { provider: recordingProvider } = createRecordingProvider();
     let embeddingCallsStarted = 0;
     let releaseConcurrentCalls!: () => void;
@@ -1012,11 +1012,23 @@ describe("Semantic Embedding Pipeline", () => {
     });
     assert.deepStrictEqual(second, { embedded: 1, skipped: 0 });
     assert.deepStrictEqual(
-      incrementalPhases.filter(
-        (phaseName) => phaseName === "hnsw.drop" || phaseName === "hnsw.create",
+      incrementalPhases.filter((phaseName) =>
+        [
+          "checkpoint.pre",
+          "hnsw.drop",
+          "inference",
+          "hnsw.create",
+          "checkpoint.post",
+        ].includes(phaseName),
       ),
-      [],
-      "incremental Symbol writes must retain the live HNSW",
+      [
+        "checkpoint.pre",
+        "hnsw.drop",
+        "inference",
+        "hnsw.create",
+        "checkpoint.post",
+      ],
+      "incremental Symbol writes must run with the shared-table HNSW absent",
     );
     assert.strictEqual(await countSymbolVectorRows(conn), rowCountBefore);
 

tests/unit/semantic-pipeline-regressions.test.ts

@@ -89,7 +89,7 @@ describe("semantic pipeline regressions", () => {
     );
   });
 
-  it("retains Symbol HNSW while preserving FileSummary rebuilds", () => {
+  it("rebuilds Symbol HNSW around incremental writes", () => {
     const symbolSource = readSource("src/indexer/embeddings.ts");
     const symbolStart = symbolSource.indexOf(
       "export async function refreshSymbolEmbeddings(",
@@ -101,40 +101,40 @@ describe("semantic pipeline regressions", () => {
       symbolEnd === -1 ? symbolSource.length : symbolEnd,
     );
 
-    assert.ok(
-      !/dropVectorIndex\s*\(/.test(symbolBody),
-      "Symbol embedding refresh must not drop its live HNSW",
-    );
-    assert.ok(
-      !/rebuildMinUncachedRows|SYMBOL_VECTOR_REBUILD_MIN_ROWS|VECTOR_REBUILD_THRESHOLD|useRebuildPath/.test(
-        symbolBody,
-      ),
-      "Symbol embedding refresh must not defer small incremental writes behind a rebuild threshold",
+    assert.match(
+      symbolBody,
+      /dropVectorIndex\(\s*wConn,\s*SYMBOL_VECTOR_EMBEDDING_TABLE,\s*indexName/s,
+      "Symbol embedding refresh must remove the shared-table HNSW before writes",
     );
-
-    const fileSummarySource = readSource(
-      "src/indexer/file-summary-embeddings.ts",
+    assert.match(
+      symbolBody,
+      /symbol-vector-rebuild-pre-drop[\s\S]*symbol-vector-rebuild-post-create/,
+      "Symbol embedding refresh must checkpoint the drop/write/recreate cycle",
     );
-    assert.match(fileSummarySource, /rebuildMinUncachedRows/);
     assert.match(
-      fileSummarySource,
-      /dropVectorIndex\(wConn, "FileSummary", indexName\)/,
-      "FileSummary keeps its existing drop/rebuild lifecycle",
+      symbolBody,
+      /dropResult\.status === "failed"[\s\S]*throw new IndexError/,
+      "Symbol embedding refresh must fail before writes when HNSW drop fails",
+    );
+    assert.doesNotMatch(
+      symbolBody,
+      /rebuildMinUncachedRows|SYMBOL_VECTOR_REBUILD_MIN_ROWS/,
+      "the safety lifecycle must not defer changed Symbol vectors",
     );
   });
 
   it("fails refresh when a required Symbol HNSW bootstrap fails", () => {
     const source = readSource("src/indexer/embeddings.ts");
-    const bootstrapStart = source.indexOf(
-      "const bootstrapVectorIndex = async (): Promise<void> =>",
+    const createStart = source.indexOf(
+      "const createRequiredVectorIndex = async (): Promise<void> =>",
     );
-    const bootstrapEnd = source.indexOf(
-      "if (storageModel === \"mock-fallback\")",
-      bootstrapStart,
+    const createEnd = source.indexOf(
+      "const bootstrapVectorIndex = async (): Promise<void> =>",
+      createStart,
     );
-    const bootstrapBody = source.slice(bootstrapStart, bootstrapEnd);
+    const createBody = source.slice(createStart, createEnd);
 
-    assert.match(bootstrapBody, /if \(!ok\)[\s\S]*throw new IndexError/);
+    assert.match(createBody, /if \(!ok\)[\s\S]*throw new IndexError/);
   });
 
   it("runs semantic rebuilds outside ambient indexer sessions", () => {