From 904094ff0b225ec486d94916a4cb1be99e0af3d0 Mon Sep 17 00:00:00 2001 From: Michael Burr Date: Mon, 2 Nov 2026 08:01:35 +0000 Subject: [PATCH 1/2] perf(search): WIP rebuild docs_fts with per-site pid token and rowid map Work in progress from an interrupted run (rate limit); not yet reviewed or fully tested. Migration 0064, lib/fts.ts rowid-map writes, trash/search/reindex call sites, and a re-runnable repair script. --- packages/admin/src/lib/projectDelete.ts | 3 +- packages/admin/src/routes/projects.test.ts | 22 +- packages/admin/src/routes/projects.ts | 10 +- .../api/migrations/0064_rebuild_docs_fts.sql | 47 ++++ packages/api/scripts/repair-docs-fts.sql | 65 +++++ packages/api/src/lib/fts.test.ts | 261 +++++++++++++++++- packages/api/src/lib/fts.ts | 133 ++++++++- packages/api/src/lib/trash.test.ts | 14 +- packages/api/src/lib/trash.ts | 18 +- packages/api/src/routes/projects.test.ts | 20 +- packages/api/src/routes/projects.ts | 13 +- packages/api/src/routes/search.test.ts | 74 ++++- packages/api/src/routes/search.ts | 19 +- 13 files changed, 647 insertions(+), 52 deletions(-) create mode 100644 packages/api/migrations/0064_rebuild_docs_fts.sql create mode 100644 packages/api/scripts/repair-docs-fts.sql diff --git a/packages/admin/src/lib/projectDelete.ts b/packages/admin/src/lib/projectDelete.ts index 7d2787fe..120ed189 100644 --- a/packages/admin/src/lib/projectDelete.ts +++ b/packages/admin/src/lib/projectDelete.ts @@ -1,4 +1,5 @@ import { releaseCustomDomain } from "../../../api/src/lib/customDomains"; +import { ftsDeleteProjectStatements } from "../../../api/src/lib/fts"; import { deleteR2Prefix } from "./r2"; import type { Env } from "../index"; @@ -65,7 +66,7 @@ export async function deleteProjectCascade( await releaseCustomDomain(env, projectId); await env.DB.batch([ - env.DB.prepare("DELETE FROM docs_fts WHERE project_id = ?").bind(projectId), + ...ftsDeleteProjectStatements(env.DB, projectId), env.DB.prepare("DELETE FROM projects WHERE id = ?").bind(projectId), ]); diff --git a/packages/admin/src/routes/projects.test.ts b/packages/admin/src/routes/projects.test.ts index be0a5a64..753f0218 100644 --- a/packages/admin/src/routes/projects.test.ts +++ b/packages/admin/src/routes/projects.test.ts @@ -2,7 +2,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { Hono } from "hono"; vi.mock("../audit", () => ({ writeAdminAudit: vi.fn() })); -vi.mock("../../../api/src/lib/fts", () => ({ upsertFtsRow: vi.fn() })); vi.mock("../../../api/src/lib/customDomains", () => ({ releaseCustomDomain: vi.fn(), removeCustomDomain: vi.fn(), @@ -304,10 +303,14 @@ describe("DELETE /:id - scale safety (AB-H1)", () => { const chunkBinds = batches.slice(0, 3).flatMap(b => b[0].binds); expect(chunkBinds).toEqual(docIds.map(d => d.id)); + // FTS rows (map first, then the rows, both via the site's pid token) go + // in the same batch as the project row. const finalBatch = batches[3]; - expect(finalBatch[0].sql).toContain("docs_fts"); - expect(finalBatch[1].sql).toContain("DELETE FROM projects"); - expect(finalBatch[1].binds).toEqual(["proj-1"]); + expect(finalBatch[0].sql).toContain("DELETE FROM docs_fts_map"); + expect(finalBatch[1].sql).toContain("DELETE FROM docs_fts WHERE docs_fts MATCH ?"); + expect(finalBatch[1].binds[0]).toMatch(/^pid:"[0-9A-F]+"$/); + expect(finalBatch[2].sql).toContain("DELETE FROM projects"); + expect(finalBatch[2].binds).toEqual(["proj-1"]); expect(releaseCustomDomain).toHaveBeenCalledWith(expect.anything(), "proj-1"); expect(writeAdminAudit).toHaveBeenCalledTimes(1); @@ -383,12 +386,21 @@ describe("POST /:id/reindex (AB-H2)", () => { }); it("reindexes a small project and audits after success", async () => { - const { db, firstQueue, allQueue } = makeDb(); + const dbRef = makeDb(); + const { db, firstQueue, allQueue } = dbRef; firstQueue.push({ id: "proj-1" }); allQueue.push([{ id: "d1", title: "One" }, { id: "d2", title: "Two" }]); const request = makeApp({ DB: db, ASSETS: r2 }); const res = await request("/proj-1/reindex", { method: "POST" }); expect(res.status).toBe(200); + // One batch for the chunk (3 statements per doc), then the stale-row prune. + const { batches } = dbRef; + expect(batches).toHaveLength(2); + expect(batches[0]).toHaveLength(6); + expect(batches[0][1].sql).toContain("INSERT INTO docs_fts"); + expect(batches[0][1].binds).toContain("d1"); + expect(batches[0][4].binds).toContain("d2"); + expect(batches[1][1].sql).toContain("doc_id NOT IN (SELECT id FROM docs WHERE project_id = ?)"); expect(writeAdminAudit).toHaveBeenCalledWith( expect.anything(), session, "project.reindex", "project", "proj-1", { indexed: 2 }, ); diff --git a/packages/admin/src/routes/projects.ts b/packages/admin/src/routes/projects.ts index 504f058b..e827205d 100644 --- a/packages/admin/src/routes/projects.ts +++ b/packages/admin/src/routes/projects.ts @@ -1,5 +1,5 @@ import { Hono } from "hono"; -import { upsertFtsRow } from "../../../api/src/lib/fts"; +import { ftsUpsertStatements, ftsPruneProjectStatements } from "../../../api/src/lib/fts"; import { removeCustomDomain } from "../../../api/src/lib/customDomains"; import { writeAdminAudit } from "../audit"; import { type KeysetCursor, encodeCursor, decodeCursor, keysetClause } from "../lib/cursor"; @@ -151,18 +151,22 @@ projectsRouter.post("/:id/reindex", async (c) => { }, 409); } + // Each doc's FTS row is replaced by rowid; one D1 batch per chunk (3 + // statements per doc), then rows of docs no longer in the site are pruned. let indexed = 0; for (let i = 0; i < docs.results.length; i += REINDEX_CHUNK) { const chunk = docs.results.slice(i, i + REINDEX_CHUNK); - await Promise.all( + const stmts = await Promise.all( chunk.map(async (doc) => { const obj = await c.env.ASSETS.get(`${projectId}/${doc.id}`); const content = obj ? await obj.text() : ""; - await upsertFtsRow(c.env.DB, doc.id, projectId, doc.title, content); + return ftsUpsertStatements(c.env.DB, doc.id, projectId, doc.title, content); }), ); + await c.env.DB.batch(stmts.flat()); indexed += chunk.length; } + await c.env.DB.batch(ftsPruneProjectStatements(c.env.DB, projectId)); await writeAdminAudit(c.env, session, "project.reindex", "project", projectId, { indexed }); return c.json({ ok: true, data: { indexed } }); diff --git a/packages/api/migrations/0064_rebuild_docs_fts.sql b/packages/api/migrations/0064_rebuild_docs_fts.sql new file mode 100644 index 00000000..5e4599fd --- /dev/null +++ b/packages/api/migrations/0064_rebuild_docs_fts.sql @@ -0,0 +1,47 @@ +-- Rebuild docs_fts so per-site search and per-doc writes stop scanning the +-- whole platform's FTS corpus (see packages/api/src/lib/fts.ts). +-- +-- 1. `pid` is a new INDEXED column holding one token per row: hex(project_id). +-- Search ANDs `pid:""` into the MATCH, so FTS5 only evaluates (and +-- runs bm25/snippet for) rows of the requested site. hex() of the UTF-8 id +-- is injective, a single token for any id under the ascii tokenizer, and +-- (72 chars for a UUID, > the porter stemmer's 64-char cap) never stemmed. +-- pid is declared after body, so snippet(docs_fts, 1, ...) is still the body +-- and bm25(docs_fts, 5.0, 1.0, 0.0) keeps the title/body weights while +-- giving the pid hit no weight. +-- 2. docs_fts_map(rid, doc_id) maps each doc to its FTS rowid, so writes are +-- rowid lookups instead of a `WHERE doc_id = ?` scan of an UNINDEXED column. +-- Invariant (kept by lib/fts.ts and lib/trash.ts): exactly one map row per +-- docs_fts row. +-- +-- Rows are copied with their rowids preserved. Only one row per doc survives +-- (the newest, should a race have left duplicates), and rows whose doc no +-- longer exists in `docs` are dropped: a binned doc's FTS row already lives in +-- trash_items.extra_json (restore re-inserts it from there), so anything left +-- in docs_fts for a missing doc is a leak (e.g. a site deleted without its +-- FTS cleanup). + +CREATE VIRTUAL TABLE docs_fts_new USING fts5( + title, + body, + pid, + doc_id UNINDEXED, + project_id UNINDEXED, + tokenize = 'porter ascii' +); + +INSERT INTO docs_fts_new (rowid, title, body, pid, doc_id, project_id) +SELECT f.rowid, f.title, f.body, hex(f.project_id), f.doc_id, f.project_id +FROM docs_fts f +WHERE f.rowid IN (SELECT max(rowid) FROM docs_fts GROUP BY doc_id) + AND f.doc_id IN (SELECT id FROM docs); + +DROP TABLE docs_fts; +ALTER TABLE docs_fts_new RENAME TO docs_fts; + +CREATE TABLE docs_fts_map ( + rid INTEGER PRIMARY KEY, + doc_id TEXT NOT NULL UNIQUE +); + +INSERT INTO docs_fts_map (rid, doc_id) SELECT rowid, doc_id FROM docs_fts; diff --git a/packages/api/scripts/repair-docs-fts.sql b/packages/api/scripts/repair-docs-fts.sql new file mode 100644 index 00000000..0445115a --- /dev/null +++ b/packages/api/scripts/repair-docs-fts.sql @@ -0,0 +1,65 @@ +-- Re-runnable repair for docs_fts / docs_fts_map (migration 0064). +-- +-- Run it once right after the api + admin workers that understand 0064 are +-- fully deployed. Between applying 0064 and that deploy, still-running old code +-- writes docs_fts rows with no `pid` and no map row, and deletes rows without +-- touching the map. Search ignores those rows (they can't match `pid:`), so the +-- only symptom is that the affected docs are missing from search until this +-- runs. Safe to run any number of times; on a consistent index it changes +-- nothing. +-- +-- cd packages/api && npx wrangler d1 execute cubedocs-main --remote --file scripts/repair-docs-fts.sql +-- +-- Old code always deletes EVERY row of a doc before inserting its new one, so +-- if a doc has both a mapped row and unmapped rows, the mapped row was written +-- later (by new code) and is the one to keep. + +-- 1. Rows whose doc no longer exists (binned docs keep theirs in +-- trash_items.extra_json; anything else is a leak). +DELETE FROM docs_fts WHERE rowid IN ( + SELECT rowid FROM docs_fts WHERE doc_id NOT IN (SELECT id FROM docs) +); + +-- 2. Map rows that don't point at their own doc's row. +DELETE FROM docs_fts_map WHERE NOT EXISTS ( + SELECT 1 FROM docs_fts f WHERE f.rowid = docs_fts_map.rid AND f.doc_id = docs_fts_map.doc_id +); + +-- 3. Unmapped rows of a doc that still has a mapped row: stale. +DELETE FROM docs_fts WHERE rowid IN ( + SELECT f.rowid FROM docs_fts f + WHERE f.rowid NOT IN (SELECT rid FROM docs_fts_map) + AND f.doc_id IN (SELECT doc_id FROM docs_fts_map) +); + +-- 4. Remaining unmapped rows: keep only the newest per doc. +DELETE FROM docs_fts WHERE rowid IN ( + SELECT rowid FROM docs_fts + WHERE rowid NOT IN (SELECT rid FROM docs_fts_map) + AND rowid NOT IN ( + SELECT max(rowid) FROM docs_fts WHERE rowid NOT IN (SELECT rid FROM docs_fts_map) GROUP BY doc_id + ) +); + +-- 5. Map them. +INSERT INTO docs_fts_map (rid, doc_id) +SELECT rowid, doc_id FROM docs_fts WHERE rowid NOT IN (SELECT rid FROM docs_fts_map); + +-- 6. Give rows written by old code their site token. +UPDATE docs_fts SET pid = hex(project_id) WHERE pid IS NULL OR pid <> hex(project_id); + +-- Verification: fts_rows = map_rows, bad_pid = 0. unindexed_docs counts docs +-- with no FTS row at all (normally 0; e.g. a doc binned by new code while its +-- row was still an unmapped old-code row, then restored). If it isn't 0, +-- reindex those sites (admin Sites -> Reindex, or the owner's +-- POST /projects/:id/reindex) - the per-site breakdown is below. +SELECT + (SELECT count(*) FROM docs_fts) AS fts_rows, + (SELECT count(*) FROM docs_fts_map) AS map_rows, + (SELECT count(*) FROM docs_fts WHERE pid IS NULL OR pid <> hex(project_id)) AS bad_pid, + (SELECT count(*) FROM docs d WHERE NOT EXISTS (SELECT 1 FROM docs_fts_map m WHERE m.doc_id = d.id)) AS unindexed_docs; + +SELECT d.project_id, count(*) AS unindexed_docs +FROM docs d +WHERE NOT EXISTS (SELECT 1 FROM docs_fts_map m WHERE m.doc_id = d.id) +GROUP BY d.project_id; diff --git a/packages/api/src/lib/fts.test.ts b/packages/api/src/lib/fts.test.ts index fb161bfb..6e0342c2 100644 --- a/packages/api/src/lib/fts.test.ts +++ b/packages/api/src/lib/fts.test.ts @@ -1,5 +1,13 @@ -import { describe, it, expect } from "vitest"; -import { sanitizeFtsQuery } from "./fts"; +import { describe, it, expect, beforeEach } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createSqliteD1 } from "../test/sqliteD1"; +import { + sanitizeFtsQuery, ftsSiteToken, siteFtsMatch, upsertFtsRow, ftsUpsertStatements, deleteFtsRow, + deleteFtsForProject, ftsPruneProjectStatements, +} from "./fts"; describe("sanitizeFtsQuery", () => { it("wraps each word in double quotes and prefix-stars the last", () => { @@ -59,3 +67,252 @@ describe("sanitizeFtsQuery", () => { expect(sanitizeFtsQuery("docs")).toBe('"docs"*'); }); }); + +// --------------------------------------------------------------------------- +// Real SQLite (node:sqlite, FTS5) against the migrated schema. +// --------------------------------------------------------------------------- + +const T0 = "2026-10-01T00:00:00.000Z"; +const P1 = "11111111-2222-3333-4444-555555555555"; +const P2 = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + +function seedSites(sqlite: DatabaseSync) { + for (const id of [P1, P2]) { + sqlite.prepare("INSERT INTO projects (id, name, owner_id, created_at) VALUES (?, ?, 'u1', ?)").run(id, id, T0); + } +} +function seedDoc(sqlite: DatabaseSync, id: string, projectId: string, title = `Title ${id}`) { + sqlite.prepare( + "INSERT INTO docs (id, title, project_id, author_id, created_at, updated_at) VALUES (?, ?, ?, 'u1', ?, ?)", + ).run(id, title, projectId, T0, T0); +} +const all = (sqlite: DatabaseSync, sql: string, ...args: (string | number | null)[]) => + sqlite.prepare(sql).all(...args).map(r => ({ ...r })); + +// The search route's doc query, minus the joins and the belt-and-braces +// `f.project_id = ?` re-check: site isolation must come from the MATCH alone. +function search(sqlite: DatabaseSync, projectId: string, q: string) { + return all( + sqlite, + `SELECT f.doc_id, snippet(docs_fts, 1, '', '', '...', 24) AS excerpt, + bm25(docs_fts, 5.0, 1.0, 0.0) AS rank + FROM docs_fts f WHERE docs_fts MATCH ? ORDER BY rank`, + siteFtsMatch(projectId, q), + ); +} + +function consistency(sqlite: DatabaseSync) { + const n = (sql: string) => Number(sqlite.prepare(sql).get()!.n); + return { + fts: n("SELECT count(*) AS n FROM docs_fts"), + map: n("SELECT count(*) AS n FROM docs_fts_map"), + linked: n("SELECT count(*) AS n FROM docs_fts_map m JOIN docs_fts f ON f.rowid = m.rid AND f.doc_id = m.doc_id"), + }; +} + +describe("ftsSiteToken / siteFtsMatch", () => { + it("matches SQLite's hex() for ASCII and non-ASCII ids", () => { + const { sqlite } = createSqliteD1(); + for (const id of [P1, "p1", "sité-ü-漢"]) { + expect(ftsSiteToken(id)).toBe(sqlite.prepare("SELECT hex(?) AS h").get(id)!.h); + } + }); + + it("scopes to the site token and column-filters every user term", () => { + expect(siteFtsMatch("p1", "hello world")).toBe('pid:"7031" AND {title body}:"hello" AND {title body}:"world"*'); + // User input can't smuggle in a column filter or operator: each word is + // a quoted phrase under the title/body filter. + expect(siteFtsMatch("p1", "pid:x OR {title}")).toBe( + 'pid:"7031" AND {title body}:"pid" AND {title body}:"x" AND {title body}:"OR" AND {title body}:"{title}"*', + ); + expect(siteFtsMatch("p1", " ")).toBe('pid:"7031" AND {title body}:""'); + }); +}); + +describe("docs_fts writes and site-scoped search (real SQLite)", () => { + let db: D1Database; + let sqlite: DatabaseSync; + + beforeEach(() => { + ({ db, sqlite } = createSqliteD1()); + seedSites(sqlite); + seedDoc(sqlite, "a1", P1); + seedDoc(sqlite, "a2", P1); + seedDoc(sqlite, "b1", P2); + }); + + it("only returns the requested site's docs", async () => { + await upsertFtsRow(db, "a1", P1, "Dragons", "The **red** dragon sleeps"); + await upsertFtsRow(db, "a2", P1, "Notes", "nothing here"); + await upsertFtsRow(db, "b1", P2, "Dragons", "A blue dragon"); + expect(search(sqlite, P1, "drag").map(r => r.doc_id)).toEqual(["a1"]); + expect(search(sqlite, P2, "drag").map(r => r.doc_id)).toEqual(["b1"]); + expect(search(sqlite, P1, "drag")[0].excerpt).toBe("The red dragon sleeps"); + }); + + it("runs operator-looking and empty input as plain terms without erroring", async () => { + await upsertFtsRow(db, "a1", P1, "Plain", "nothing to see"); + for (const q of ["pid:x OR {title}", "NEAR(a b)", "a AND NOT b", ".", " ", '"unterminated']) { + expect(() => search(sqlite, P1, q)).not.toThrow(); + } + expect(search(sqlite, P1, " ")).toEqual([]); + }); + + it("never matches the site token itself, even on a prefix of it", async () => { + await upsertFtsRow(db, "a1", P1, "Plain", "nothing to see"); + // P1's token starts "3131..."; a bare `"3"*` over every column would hit it. + expect(ftsSiteToken(P1).startsWith("3")).toBe(true); + expect(search(sqlite, P1, "3")).toEqual([]); + expect(search(sqlite, P1, ftsSiteToken(P1))).toEqual([]); + }); + + it("replaces a doc's row on update instead of duplicating it", async () => { + await upsertFtsRow(db, "a1", P1, "First", "alpha"); + await upsertFtsRow(db, "a1", P1, "Second", "beta"); + await upsertFtsRow(db, "a1", P1, "Third", "gamma"); + expect(all(sqlite, "SELECT doc_id, title, body, pid, project_id FROM docs_fts")).toEqual([ + { doc_id: "a1", title: "Third", body: "gamma", pid: ftsSiteToken(P1), project_id: P1 }, + ]); + expect(consistency(sqlite)).toEqual({ fts: 1, map: 1, linked: 1 }); + expect(search(sqlite, P1, "alpha")).toEqual([]); + expect(search(sqlite, P1, "gamma").map(r => r.doc_id)).toEqual(["a1"]); + }); + + it("ftsUpsertStatements composes into a caller's batch", async () => { + await db.batch([ + ...ftsUpsertStatements(db, "a1", P1, "One", "first"), + ...ftsUpsertStatements(db, "a2", P1, "Two", "second"), + ...ftsUpsertStatements(db, "a1", P1, "One", "first again"), + ]); + expect(all(sqlite, "SELECT doc_id, body FROM docs_fts ORDER BY doc_id")).toEqual([ + { doc_id: "a1", body: "first again" }, + { doc_id: "a2", body: "second" }, + ]); + expect(consistency(sqlite)).toEqual({ fts: 2, map: 2, linked: 2 }); + }); + + it("deleteFtsRow removes one doc's row and map entry only", async () => { + await upsertFtsRow(db, "a1", P1, "One", "x"); + await upsertFtsRow(db, "a2", P1, "Two", "x"); + await deleteFtsRow(db, "a1"); + await deleteFtsRow(db, "never-indexed"); + expect(all(sqlite, "SELECT doc_id FROM docs_fts")).toEqual([{ doc_id: "a2" }]); + expect(consistency(sqlite)).toEqual({ fts: 1, map: 1, linked: 1 }); + }); + + it("deleteFtsForProject removes only that site's rows and map entries", async () => { + await upsertFtsRow(db, "a1", P1, "One", "x"); + await upsertFtsRow(db, "a2", P1, "Two", "x"); + await upsertFtsRow(db, "b1", P2, "Three", "x"); + await deleteFtsForProject(db, P1); + expect(all(sqlite, "SELECT doc_id FROM docs_fts")).toEqual([{ doc_id: "b1" }]); + expect(all(sqlite, "SELECT doc_id FROM docs_fts_map")).toEqual([{ doc_id: "b1" }]); + }); + + it("the post-reindex prune drops only the site's rows for docs no longer in it", async () => { + await upsertFtsRow(db, "a1", P1, "One", "x"); + await upsertFtsRow(db, "a2", P1, "Two", "x"); + await upsertFtsRow(db, "b1", P2, "Three", "x"); + sqlite.exec("DELETE FROM docs WHERE id IN ('a2', 'b1')"); + await db.batch(ftsPruneProjectStatements(db, P1)); + expect(all(sqlite, "SELECT doc_id FROM docs_fts ORDER BY doc_id")).toEqual([{ doc_id: "a1" }, { doc_id: "b1" }]); + expect(consistency(sqlite)).toEqual({ fts: 2, map: 2, linked: 2 }); + }); + + it("tolerates rows written by pre-0064 code (no pid, no map row) without touching other docs", async () => { + await upsertFtsRow(db, "a1", P1, "One", "kept"); + const a1Rid = Number(sqlite.prepare("SELECT rid FROM docs_fts_map WHERE doc_id = 'a1'").get()!.rid); + // Old code: `DELETE ... WHERE doc_id = ?` + plain INSERT. Delete a1's row + // and put b1's old-style row at a1's old rowid, so a1's map row is stale. + sqlite.exec("DELETE FROM docs_fts WHERE doc_id = 'a1'"); + sqlite.prepare("INSERT INTO docs_fts (rowid, doc_id, project_id, title, body) VALUES (?, 'b1', ?, 'Old', 'legacy')") + .run(a1Rid, P2); + // Invisible to site-scoped search until repaired (no pid)... + expect(search(sqlite, P2, "legacy")).toEqual([]); + // ...and a1's upsert must not delete b1's row through the stale map entry. + await upsertFtsRow(db, "a1", P1, "One", "fresh"); + expect(all(sqlite, "SELECT doc_id, body FROM docs_fts ORDER BY doc_id")).toEqual([ + { doc_id: "a1", body: "fresh" }, + { doc_id: "b1", body: "legacy" }, + ]); + expect(all(sqlite, "SELECT doc_id FROM docs_fts_map")).toEqual([{ doc_id: "a1" }]); + }); +}); + +describe("scripts/repair-docs-fts.sql", () => { + it("reconciles rows written by pre-0064 code and is idempotent", async () => { + const { db, sqlite } = createSqliteD1(); + seedSites(sqlite); + for (const [id, p] of [["a1", P1], ["a2", P1], ["a3", P1], ["b1", P2]] as const) seedDoc(sqlite, id, p); + await upsertFtsRow(db, "a2", P1, "A2", "will be rewritten by old code"); + await upsertFtsRow(db, "b1", P2, "B1", "untouched"); + const oldInsert = sqlite.prepare("INSERT INTO docs_fts (doc_id, project_id, title, body) VALUES (?, ?, ?, ?)"); + // a1: an old-code row, then a later new-code save -> the old row is stale. + oldInsert.run("a1", P1, "A1", "stale old-code body"); + await upsertFtsRow(db, "a1", P1, "A1", "mapped current"); + // a2: rewritten by old code -> its map row is stale, the new row unmapped. + sqlite.exec("DELETE FROM docs_fts WHERE doc_id = 'a2'"); + oldInsert.run("a2", P1, "A2", "old code wrote this last"); + // a3: only ever written by old code (two racing inserts). + oldInsert.run("a3", P1, "A3", "first"); + oldInsert.run("a3", P1, "A3", "second"); + // An orphan for a doc that no longer exists. + oldInsert.run("gone", P1, "Gone", "orphan"); + + const repair = readFileSync(fileURLToPath(new URL("../../scripts/repair-docs-fts.sql", import.meta.url)), "utf8"); + const snapshot = () => all(sqlite, "SELECT doc_id, body, pid FROM docs_fts ORDER BY doc_id"); + sqlite.exec(repair); + const once = snapshot(); + expect(once).toEqual([ + { doc_id: "a1", body: "mapped current", pid: ftsSiteToken(P1) }, + { doc_id: "a2", body: "old code wrote this last", pid: ftsSiteToken(P1) }, + { doc_id: "a3", body: "second", pid: ftsSiteToken(P1) }, + { doc_id: "b1", body: "untouched", pid: ftsSiteToken(P2) }, + ]); + expect(consistency(sqlite)).toEqual({ fts: 4, map: 4, linked: 4 }); + expect(search(sqlite, P1, "old code").map(r => r.doc_id)).toEqual(["a2"]); + + sqlite.exec(repair); + expect(snapshot()).toEqual(once); + expect(consistency(sqlite)).toEqual({ fts: 4, map: 4, linked: 4 }); + }); +}); + +describe("migration 0064_rebuild_docs_fts", () => { + const dir = fileURLToPath(new URL("../../migrations", import.meta.url)); + const before = readdirSync(dir).filter(f => f.endsWith(".sql") && f < "0064_").sort(); + + it("copies live rows with rowids and contents preserved, drops orphans and duplicates, builds the map", () => { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA foreign_keys = OFF"); + for (const f of before) sqlite.exec(readFileSync(join(dir, f), "utf8")); + seedSites(sqlite); + for (const [id, p] of [["d1", P1], ["d2", P1], ["d3", P2], ["d4", P2]] as const) seedDoc(sqlite, id, p); + const ins = sqlite.prepare("INSERT INTO docs_fts (rowid, doc_id, project_id, title, body) VALUES (?, ?, ?, ?, ?)"); + ins.run(5, "d1", P1, "Dragon lore", "red dragons and more"); + ins.run(9, "d2", P1, "Notes", "plain notes"); + ins.run(12, "d3", P2, "Dragon facts", "blue dragon"); + ins.run(14, "d4", P2, "Dup", "older duplicate"); + ins.run(17, "d4", P2, "Dup", "newer duplicate"); + ins.run(20, "binned-or-leaked", P1, "Gone", "orphan row"); + + sqlite.exec(readFileSync(join(dir, "0064_rebuild_docs_fts.sql"), "utf8")); + + expect(all(sqlite, "SELECT rowid AS r, doc_id, project_id, title, body, pid FROM docs_fts ORDER BY rowid")).toEqual([ + { r: 5, doc_id: "d1", project_id: P1, title: "Dragon lore", body: "red dragons and more", pid: ftsSiteToken(P1) }, + { r: 9, doc_id: "d2", project_id: P1, title: "Notes", body: "plain notes", pid: ftsSiteToken(P1) }, + { r: 12, doc_id: "d3", project_id: P2, title: "Dragon facts", body: "blue dragon", pid: ftsSiteToken(P2) }, + { r: 17, doc_id: "d4", project_id: P2, title: "Dup", body: "newer duplicate", pid: ftsSiteToken(P2) }, + ]); + expect(all(sqlite, "SELECT rid, doc_id FROM docs_fts_map ORDER BY rid")).toEqual([ + { rid: 5, doc_id: "d1" }, { rid: 9, doc_id: "d2" }, { rid: 12, doc_id: "d3" }, { rid: 17, doc_id: "d4" }, + ]); + // No temp table or its shadow tables left behind. + expect(all(sqlite, "SELECT name FROM sqlite_master WHERE name LIKE 'docs_fts_new%'")).toEqual([]); + // The rebuilt index serves site-scoped search, with the usual snippet. + expect(search(sqlite, P1, "dragon")).toEqual([ + { doc_id: "d1", excerpt: "red dragons and more", rank: expect.any(Number) }, + ]); + expect(search(sqlite, P2, "dragon").map(r => r.doc_id)).toEqual(["d3"]); + }); +}); diff --git a/packages/api/src/lib/fts.ts b/packages/api/src/lib/fts.ts index f0b1fc5f..c2838240 100644 --- a/packages/api/src/lib/fts.ts +++ b/packages/api/src/lib/fts.ts @@ -17,12 +17,92 @@ function stripMarkdown(content: string): string { return text.replace(/\s+/g, " ").trim(); } -export function sanitizeFtsQuery(q: string): string { +// Each user word as a quoted FTS5 phrase. Quotes are stripped first, so a word +// can never close its phrase: user input can only ever be plain terms, never +// operators or column filters. +function ftsTerms(q: string): string[] { const words = q.replace(/['"*()^:~\-]/g, " ").trim().split(/\s+/).filter(Boolean); - if (!words.length) return '""'; // Star the final token so search-as-you-type matches partially typed words // ("colla" finds "collaboration"); earlier tokens must be complete words. - return words.map((w, i) => (i === words.length - 1 ? `"${w}"*` : `"${w}"`)).join(" "); + return words.map((w, i) => (i === words.length - 1 ? `"${w}"*` : `"${w}"`)); +} + +export function sanitizeFtsQuery(q: string): string { + const terms = ftsTerms(q); + return terms.length ? terms.join(" ") : '""'; +} + +// --------------------------------------------------------------------------- +// docs_fts layout (migrations/0064_rebuild_docs_fts.sql): +// +// docs_fts(title, body, pid, doc_id UNINDEXED, project_id UNINDEXED) +// docs_fts_map(rid INTEGER PRIMARY KEY, doc_id TEXT UNIQUE) +// +// `pid` holds one indexed token per row, hex(project_id), so a search ANDs +// `pid:""` into its MATCH and FTS5 only evaluates that site's rows. +// docs_fts_map gives every doc its FTS rowid, so per-doc writes are rowid +// lookups rather than a scan of the UNINDEXED doc_id column. Every writer +// below keeps exactly one map row per docs_fts row. +// +// A doc's row is replaced, not updated in place: the old row (found through +// the map, and only if it really is this doc's) is deleted, the new one is +// inserted with an automatic rowid (max + 1, so it can never collide with a +// row some other writer put there), and the map is pointed at it via +// last_insert_rowid(). INSERT OR REPLACE also drops any stale map row that +// claimed the new rowid. The three statements must stay adjacent in a batch. +// --------------------------------------------------------------------------- + +// The `pid` token for a site: hex of the id's UTF-8 bytes, exactly what +// SQLite's hex(project_id) produces (the migration and the writers below use +// that). Injective, a single token for any id under the ascii tokenizer, and +// longer than the porter stemmer's 64-char cap for UUIDs. +export function ftsSiteToken(projectId: string): string { + let out = ""; + for (const b of new TextEncoder().encode(projectId)) out += b.toString(16).padStart(2, "0"); + return out.toUpperCase(); +} + +// MATCH expression for a site-scoped search: the site token, AND every +// sanitized user term restricted to the title/body columns. The column filter +// is what keeps a user prefix like "a"* from matching the pid token itself. +// Ranking is unchanged as long as callers weight pid 0: bm25(docs_fts, 5.0, 1.0, 0.0). +export function siteFtsMatch(projectId: string, q: string): string { + const terms = ftsTerms(q); + const filtered = terms.length ? terms.map(t => `{title body}:${t}`) : ['{title body}:""']; + return [`pid:"${ftsSiteToken(projectId)}"`, ...filtered].join(" AND "); +} + +function siteOnlyMatch(projectId: string): string { + return `pid:"${ftsSiteToken(projectId)}"`; +} + +// Inserts an already-stripped title/body as the doc's row (replacing any +// existing one). Used directly by recycle-bin restore, which re-inserts the +// exact title/body it snapshotted. +export function ftsInsertStatements( + db: D1Database, + docId: string, + projectId: string, + title: string, + body: string, +): D1PreparedStatement[] { + return [ + db.prepare("DELETE FROM docs_fts WHERE rowid = (SELECT rid FROM docs_fts_map WHERE doc_id = ?) AND doc_id = ?") + .bind(docId, docId), + db.prepare("INSERT INTO docs_fts (title, body, pid, doc_id, project_id) VALUES (?, ?, hex(?), ?, ?)") + .bind(title, body, projectId, docId, projectId), + db.prepare("INSERT OR REPLACE INTO docs_fts_map (rid, doc_id) VALUES (last_insert_rowid(), ?)").bind(docId), + ]; +} + +export function ftsUpsertStatements( + db: D1Database, + docId: string, + projectId: string, + title: string, + content: string, +): D1PreparedStatement[] { + return ftsInsertStatements(db, docId, projectId, title, stripMarkdown(content)); } export async function upsertFtsRow( @@ -32,18 +112,49 @@ export async function upsertFtsRow( title: string, content: string, ): Promise { - const body = stripMarkdown(content); - await db.batch([ - db.prepare("DELETE FROM docs_fts WHERE doc_id = ?").bind(docId), - db.prepare("INSERT INTO docs_fts(doc_id, project_id, title, body) VALUES (?, ?, ?, ?)") - .bind(docId, projectId, title, body), - ]); + await db.batch(ftsUpsertStatements(db, docId, projectId, title, content)); +} + +// Deletes the rows of up to ~90 docs (one bound parameter per id). The +// correlated doc_id check makes sure a map row can only ever delete its own +// doc's row. +export function ftsDeleteStatements(db: D1Database, docIds: string[]): D1PreparedStatement[] { + if (docIds.length === 0) return []; + const ph = docIds.map(() => "?").join(","); + return [ + db.prepare( + `DELETE FROM docs_fts WHERE rowid IN (SELECT rid FROM docs_fts_map WHERE doc_id IN (${ph})) + AND doc_id = (SELECT m.doc_id FROM docs_fts_map m WHERE m.rid = docs_fts.rowid)`, + ).bind(...docIds), + db.prepare(`DELETE FROM docs_fts_map WHERE doc_id IN (${ph})`).bind(...docIds), + ]; } export async function deleteFtsRow(db: D1Database, docId: string): Promise { - await db.prepare("DELETE FROM docs_fts WHERE doc_id = ?").bind(docId).run(); + await db.batch(ftsDeleteStatements(db, [docId])); +} + +// Every row of one site, found through the indexed pid token. Map rows first, +// while the FTS rows they point at still exist. +export function ftsDeleteProjectStatements(db: D1Database, projectId: string): D1PreparedStatement[] { + const match = siteOnlyMatch(projectId); + return [ + db.prepare("DELETE FROM docs_fts_map WHERE rid IN (SELECT rowid FROM docs_fts WHERE docs_fts MATCH ?)").bind(match), + db.prepare("DELETE FROM docs_fts WHERE docs_fts MATCH ?").bind(match), + ]; } export async function deleteFtsForProject(db: D1Database, projectId: string): Promise { - await db.prepare("DELETE FROM docs_fts WHERE project_id = ?").bind(projectId).run(); + await db.batch(ftsDeleteProjectStatements(db, projectId)); +} + +// After a full-site reindex: drops the site's rows whose doc is no longer in +// the site (e.g. left behind by an interrupted delete). +export function ftsPruneProjectStatements(db: D1Database, projectId: string): D1PreparedStatement[] { + const match = siteOnlyMatch(projectId); + const stale = "SELECT rowid FROM docs_fts WHERE docs_fts MATCH ? AND doc_id NOT IN (SELECT id FROM docs WHERE project_id = ?)"; + return [ + db.prepare(`DELETE FROM docs_fts_map WHERE rid IN (${stale})`).bind(match, projectId), + db.prepare(`DELETE FROM docs_fts WHERE rowid IN (${stale})`).bind(match, projectId), + ]; } diff --git a/packages/api/src/lib/trash.test.ts b/packages/api/src/lib/trash.test.ts index 96049436..c8314325 100644 --- a/packages/api/src/lib/trash.test.ts +++ b/packages/api/src/lib/trash.test.ts @@ -5,6 +5,7 @@ import { moveToTrash, restoreFromTrash, purgeTrashEntry, emptyTrash, purgeExpiredTrash, listTrash, TRASH_RETENTION_DAYS, } from "./trash"; import type { Env } from "../index"; +import { siteFtsMatch } from "./fts"; // Runs against real SQLite with every migration applied (see test/sqliteD1.ts): // the bin is mostly SQL - FK cascades, recursive subtree CTEs, batch @@ -46,7 +47,8 @@ function doc(id: string, folderId: string | null, extra: { slug?: string } = {}) "INSERT INTO docs (id, title, project_id, author_id, folder_id, slug, created_at, updated_at) VALUES (?, ?, 'p1', 'u1', ?, ?, ?, ?)", id, `Title ${id}`, folderId, extra.slug ?? null, T0, T0, ); - exec("INSERT INTO docs_fts (doc_id, project_id, title, body) VALUES (?, 'p1', ?, ?)", id, `Title ${id}`, `body of ${id}`); + exec("INSERT INTO docs_fts (doc_id, project_id, pid, title, body) VALUES (?, 'p1', hex('p1'), ?, ?)", id, `Title ${id}`, `body of ${id}`); + exec("INSERT INTO docs_fts_map (rid, doc_id) VALUES (last_insert_rowid(), ?)", id); r2.objects.add(`p1/${id}`); } function file(id: string, folderId: string | null, size = 100) { @@ -86,6 +88,8 @@ describe("moveToTrash", () => { expect(rows("SELECT id FROM docs")).toEqual([{ id: "outside" }]); expect(count("files")).toBe(0); expect(rows("SELECT doc_id FROM docs_fts")).toEqual([{ doc_id: "outside" }]); + // The FTS rowid map loses the binned docs' rows with them. + expect(rows("SELECT doc_id FROM docs_fts_map")).toEqual([{ doc_id: "outside" }]); const [entry] = await listTrash(env.DB, "p1"); expect(entry).toMatchObject({ @@ -140,6 +144,7 @@ describe("restoreFromTrash", () => { folders: rows("SELECT * FROM folders ORDER BY id"), docs: rows("SELECT * FROM docs"), files: rows("SELECT * FROM files"), + fts: rows("SELECT title, body, pid, doc_id, project_id FROM docs_fts"), }; const { entryId } = (await moveToTrash(env, { projectId: "p1", type: "folder", id: "A", deletedBy: "u1" }))!; @@ -155,6 +160,13 @@ describe("restoreFromTrash", () => { expect(rows("SELECT * FROM files")).toEqual(before.files); expect(rows("SELECT id, user_id FROM doc_shares")).toEqual([{ id: "s1", user_id: "u2" }]); expect(rows("SELECT doc_id, body FROM docs_fts")).toEqual([{ doc_id: "d1", body: "body of d1" }]); + // The FTS row comes back exactly (title, body, site token), with a map row + // pointing at it, and is findable by a site-scoped search again. + expect(rows("SELECT title, body, pid, doc_id, project_id FROM docs_fts")).toEqual(before.fts); + expect(rows("SELECT m.doc_id FROM docs_fts_map m JOIN docs_fts f ON f.rowid = m.rid AND f.doc_id = m.doc_id")) + .toEqual([{ doc_id: "d1" }]); + expect(count("docs_fts_map")).toBe(1); + expect(rows("SELECT doc_id FROM docs_fts WHERE docs_fts MATCH ?", siteFtsMatch("p1", "body"))).toEqual([{ doc_id: "d1" }]); expect(rows("SELECT doc_id FROM site_reports")).toEqual([{ doc_id: "d1" }]); expect(count("trash_entries")).toBe(0); expect(count("trash_items")).toBe(0); diff --git a/packages/api/src/lib/trash.ts b/packages/api/src/lib/trash.ts index c3bded03..880a72f3 100644 --- a/packages/api/src/lib/trash.ts +++ b/packages/api/src/lib/trash.ts @@ -22,6 +22,7 @@ import type { Env } from "../index"; import { ProjectFeatures } from "../lib"; import { invalidateProjectGraphIndex } from "./docLinks"; +import { ftsDeleteStatements, ftsInsertStatements } from "./fts"; export const TRASH_RETENTION_DAYS = 30; @@ -178,7 +179,15 @@ export async function moveToTrash( (e.shares ??= []).push(s); extras.set(s.doc_id as string, e); } - for (const f of await selectIn(db, ph => `SELECT doc_id, title, body FROM docs_fts WHERE doc_id IN (${ph})`, docIds)) { + // Through docs_fts_map: rowid lookups, not a scan of the whole FTS table. + const ftsRows = await selectIn( + db, + ph => `SELECT m.doc_id, f.title, f.body FROM docs_fts_map m + JOIN docs_fts f ON f.rowid = m.rid AND f.doc_id = m.doc_id + WHERE m.doc_id IN (${ph})`, + docIds, + ); + for (const f of ftsRows) { const e = extras.get(f.doc_id as string) ?? {}; e.fts = { title: f.title as string, body: f.body as string }; extras.set(f.doc_id as string, e); @@ -233,7 +242,7 @@ export async function moveToTrash( } for (const chunk of chunked(docIds)) { - stmts.push(db.prepare(`DELETE FROM docs_fts WHERE doc_id IN (${qs(chunk.length)})`).bind(...chunk)); + stmts.push(...ftsDeleteStatements(db, chunk)); stmts.push(db.prepare(`DELETE FROM docs WHERE id IN (${qs(chunk.length)})`).bind(...chunk)); } for (const chunk of chunked(fileIds)) { @@ -348,8 +357,9 @@ export async function restoreFromTrash(env: Env, projectId: string, entryId: str .bind(...keys.map(k => share[k] ?? null))); } if (extra.fts) { - stmts.push(db.prepare("INSERT INTO docs_fts(doc_id, project_id, title, body) VALUES (?, ?, ?, ?)") - .bind(row.id as string, projectId, extra.fts.title, extra.fts.body)); + // The exact snapshotted title/body (not re-derived), as a fresh row + + // map entry. The statements must stay adjacent (last_insert_rowid()). + stmts.push(...ftsInsertStatements(db, row.id as string, projectId, extra.fts.title, extra.fts.body)); } for (const reportId of extra.reportIds ?? []) { stmts.push(db.prepare("UPDATE site_reports SET doc_id = ? WHERE id = ? AND doc_id IS NULL").bind(row.id as string, reportId)); diff --git a/packages/api/src/routes/projects.test.ts b/packages/api/src/routes/projects.test.ts index 15c008fe..51d8518d 100644 --- a/packages/api/src/routes/projects.test.ts +++ b/packages/api/src/routes/projects.test.ts @@ -3,12 +3,17 @@ import { handleProjects } from "./projects"; import { createSqliteD1 } from "../test/sqliteD1"; vi.mock("../lib/access", () => ({ resolveRole: vi.fn() })); -vi.mock("../lib/fts", () => ({ upsertFtsRow: vi.fn(), deleteFtsForProject: vi.fn() })); +vi.mock("../lib/fts", () => ({ + upsertFtsRow: vi.fn(), + deleteFtsForProject: vi.fn(), + ftsUpsertStatements: vi.fn((_db: unknown, docId: string) => [`upsert:${docId}`]), + ftsPruneProjectStatements: vi.fn(() => ["prune"]), +})); vi.mock("../lib/customDomains", () => ({ releaseCustomDomain: vi.fn() })); import { resolveRole } from "../lib/access"; import { releaseCustomDomain } from "../lib/customDomains"; -import { upsertFtsRow } from "../lib/fts"; +import { ftsUpsertStatements } from "../lib/fts"; const user = { userId: "user-1", email: "a@example.com" } as unknown as Parameters[2]; @@ -407,15 +412,20 @@ describe("handleProjects POST /projects/:id/reindex", () => { }); it("reindexes every doc and counts them", async () => { - const { env, queueAll, assetsGet } = makeEnv(); + const { env, queueAll, assetsGet, batch } = makeEnv(); queueAll({ results: [{ id: "d1", title: "A" }, { id: "d2", title: "B" }, { id: "d3", title: "C" }] }); // docs assetsGet.mockResolvedValue({ text: async () => "# body" }); // R2 content per doc const res = await call(env, "POST", "/projects/p1/reindex"); expect(res.status).toBe(200); const json = (await res.json()) as { data: { indexed: number } }; expect(json.data.indexed).toBe(3); - // One FTS upsert per doc - a dropped loop body would not satisfy this. - expect(upsertFtsRow).toHaveBeenCalledTimes(3); + // One FTS upsert per doc - a dropped loop body would not satisfy this - + // written as one batch per chunk, then one prune of stale site rows. + expect(ftsUpsertStatements).toHaveBeenCalledTimes(3); + expect(ftsUpsertStatements).toHaveBeenCalledWith(expect.anything(), "d1", "p1", "A", "# body"); + expect(batch).toHaveBeenCalledTimes(2); + expect(batch).toHaveBeenNthCalledWith(1, ["upsert:d1", "upsert:d2", "upsert:d3"]); + expect(batch).toHaveBeenNthCalledWith(2, ["prune"]); }); }); diff --git a/packages/api/src/routes/projects.ts b/packages/api/src/routes/projects.ts index 1b4b4a3b..95837966 100644 --- a/packages/api/src/routes/projects.ts +++ b/packages/api/src/routes/projects.ts @@ -1,6 +1,6 @@ import { okResponse, errorResponse, Errors, ROLE_RANK, ProjectFeatures, type Session, type Project, type Role } from "../lib"; import { deleteR2Prefix } from "../lib/docOps"; -import { upsertFtsRow, deleteFtsForProject } from "../lib/fts"; +import { upsertFtsRow, deleteFtsForProject, ftsUpsertStatements, ftsPruneProjectStatements } from "../lib/fts"; import { resolveRole } from "../lib/access"; import { releaseCustomDomain } from "../lib/customDomains"; import type { Env } from "../index"; @@ -430,17 +430,24 @@ export async function handleProjects( const rows = await env.DB.prepare("SELECT id, title FROM docs WHERE project_id = ?") .bind(projectId).all<{ id: string; title: string }>(); - const CHUNK = 50; + // Each doc's row is replaced by rowid (3 statements), 16 docs per batch to + // keep batches under ~50 statements. Replacing in place rather than + // clearing the site first means search never sees a half-empty index; + // the prune at the end drops rows of docs no longer in the site. + const CHUNK = 16; let indexed = 0; for (let i = 0; i < rows.results.length; i += CHUNK) { const chunk = rows.results.slice(i, i + CHUNK); + const stmts: D1PreparedStatement[] = []; for (const doc of chunk) { const r2 = await env.ASSETS.get(`${projectId}/${doc.id}`); const content = r2 ? await r2.text() : ""; - await upsertFtsRow(env.DB, doc.id, projectId, doc.title, content); + stmts.push(...ftsUpsertStatements(env.DB, doc.id, projectId, doc.title, content)); } + await env.DB.batch(stmts); indexed += chunk.length; } + await env.DB.batch(ftsPruneProjectStatements(env.DB, projectId)); return okResponse({ indexed }); } diff --git a/packages/api/src/routes/search.test.ts b/packages/api/src/routes/search.test.ts index 98ad4356..1e82cbc3 100644 --- a/packages/api/src/routes/search.test.ts +++ b/packages/api/src/routes/search.test.ts @@ -1,5 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import { handleSearch } from "./search"; +import { handleSearch, handlePublicSearch } from "./search"; +import { createSqliteD1 } from "../test/sqliteD1"; +import { upsertFtsRow } from "../lib/fts"; vi.mock("../lib/access", () => ({ resolveRole: vi.fn() })); @@ -80,16 +82,17 @@ describe("handleSearch", () => { expect(json.data.files[0].file_id).toBe("f1"); expect(json.data.folders).toHaveLength(1); expect(json.data.folders[0].folder_id).toBe("fo1"); - // The non-limited doc path must MATCH the FTS table AND scope by project, - // with no doc_shares join, weighting title over body. sanitizeFtsQuery - // prefix-stars the last word for search-as-you-type; bind order is - // (query, projectId). + // The non-limited doc path must MATCH the FTS table scoped to the site's + // indexed pid token (and re-check project_id), with no doc_shares filter, + // weighting title over body and pid not at all. The user's terms are + // column-filtered to title/body and the last one prefix-starred for + // search-as-you-type; bind order is (query, projectId). const docSql = prepare.mock.calls[0][0] as string; expect(docSql).toContain("docs_fts MATCH ?"); expect(docSql).toContain("f.project_id = ?"); - expect(docSql).toContain("bm25(docs_fts, 5.0, 1.0)"); + expect(docSql).toContain("bm25(docs_fts, 5.0, 1.0, 0.0)"); expect(docSql).not.toContain("doc_shares"); - expect(bind).toHaveBeenCalledWith('"hello"*', "p1"); + expect(bind).toHaveBeenCalledWith('pid:"7031" AND {title body}:"hello"*', "p1"); // The file and folder queries scope by project and LIKE-match the raw term. const fileSql = prepare.mock.calls[1][0] as string; expect(fileSql).toContain("FROM files fi"); @@ -137,8 +140,10 @@ describe("handleSearch", () => { const { env, prepare, bind, queueAll } = makeEnv(); queueAll({ results: [] }); const res = await call(env, "projectId=p1&q=hello"); - // limited path binds the userId first (for the doc_shares join) - expect(bind).toHaveBeenCalledWith("user-1", expect.anything(), "p1"); + // limited path filters the site's matches down to the caller's shares, + // read from doc_shares by its (project_id, user_id) index. + expect(prepare.mock.calls[0][0]).toContain("f.doc_id IN (SELECT doc_id FROM doc_shares WHERE project_id = ? AND user_id = ?)"); + expect(bind).toHaveBeenCalledWith(expect.stringContaining('pid:"7031"'), "p1", "p1", "user-1"); // limited members have no file/folder access, so only the doc query runs. expect(prepare).toHaveBeenCalledTimes(1); const json = (await res.json()) as SearchBody; @@ -146,3 +151,54 @@ describe("handleSearch", () => { expect(json.data.folders).toEqual([]); }); }); + +// Real SQLite (FTS5) with every migration applied: the MATCH scoping is in the +// SQL, so this is what proves one site's search can't see another's docs. +describe("doc search against real SQLite", () => { + const T0 = "2026-10-01T00:00:00.000Z"; + let env: Parameters[1]; + + beforeEach(async () => { + const { db, sqlite } = createSqliteD1(); + env = { DB: db } as unknown as Parameters[1]; + const exec = (sql: string, ...args: (string | null)[]) => sqlite.prepare(sql).run(...args); + exec("INSERT INTO projects (id, name, owner_id, created_at, published_at) VALUES ('p1', 'One', 'u1', ?, ?)", T0, T0); + exec("INSERT INTO projects (id, name, owner_id, created_at, published_at) VALUES ('p2', 'Two', 'u1', ?, ?)", T0, T0); + for (const [id, p, title] of [["a1", "p1", "Dragon lore"], ["a2", "p1", "Dragon diet"], ["b1", "p2", "Dragon facts"]]) { + exec("INSERT INTO docs (id, title, project_id, author_id, created_at, updated_at) VALUES (?, ?, ?, 'u1', ?, ?)", id, title, p, T0, T0); + } + await upsertFtsRow(db, "a1", "p1", "Dragon lore", "Old **dragons** sleep"); + await upsertFtsRow(db, "a2", "p1", "Dragon diet", "dragons eat a lot"); + await upsertFtsRow(db, "b1", "p2", "Dragon facts", "blue dragons"); + exec("INSERT INTO doc_shares (id, doc_id, user_id, project_id, granted_by, created_at) VALUES ('s1', 'a2', 'user-1', 'p1', 'u1', ?)", T0); + // A share of another site's doc must not widen a limited member's p1 search. + exec("INSERT INTO doc_shares (id, doc_id, user_id, project_id, granted_by, created_at) VALUES ('s2', 'b1', 'user-1', 'p2', 'u1', ?)", T0); + }); + + async function docIds(res: Response) { + const json = (await res.json()) as SearchBody; + return json.data.docs.map(d => d.doc_id).sort(); + } + + it("a member only sees the site's own docs", async () => { + expect(await docIds(await call(env, "projectId=p1&q=drag"))).toEqual(["a1", "a2"]); + expect(await docIds(await call(env, "projectId=p2&q=drag"))).toEqual(["b1"]); + }); + + it("returns the usual snippet", async () => { + const res = await call(env, "projectId=p1&q=sleep"); + const json = (await res.json()) as { data: { docs: Array<{ doc_id: string; excerpt: string }> } }; + expect(json.data.docs).toEqual([expect.objectContaining({ doc_id: "a1", excerpt: "Old dragons sleep" })]); + }); + + it("a limited member only sees docs shared with them in this site", async () => { + vi.mocked(resolveRole).mockResolvedValue("limited"); + expect(await docIds(await call(env, "projectId=p1&q=drag"))).toEqual(["a2"]); + }); + + it("public search is scoped to the published site", async () => { + const url = new URL("http://localhost/public/search?projectId=p2&q=drag"); + const res = await handlePublicSearch(new Request(url.toString()), env, url); + expect(await docIds(res)).toEqual(["b1"]); + }); +}); diff --git a/packages/api/src/routes/search.ts b/packages/api/src/routes/search.ts index 907ab196..58c27198 100644 --- a/packages/api/src/routes/search.ts +++ b/packages/api/src/routes/search.ts @@ -1,5 +1,5 @@ import { okResponse, errorResponse, Errors, type Session } from "../lib"; -import { sanitizeFtsQuery } from "../lib/fts"; +import { siteFtsMatch } from "../lib/fts"; import type { Env } from "../index"; import { resolveRole } from "../lib/access"; @@ -107,7 +107,10 @@ export async function handleSearch( return okResponse({ docs, files: [], folders: [] } satisfies SearchResponse); } - const ftsQuery = sanitizeFtsQuery(q!); + // The MATCH is scoped to the site by its indexed pid token, so FTS5 only + // evaluates this site's rows; `f.project_id = ?` stays as a cheap + // belt-and-braces check on the rows it returns. pid gets bm25 weight 0. + const ftsQuery = siteFtsMatch(projectId, q!); let docsPromise; if (role === "limited") { @@ -115,22 +118,22 @@ export async function handleSearch( SELECT f.doc_id, f.title, snippet(docs_fts, 1, '', '', '...', 24) AS excerpt, fo.name AS folder, d.updated_at, - bm25(docs_fts, 5.0, 1.0) AS rank + bm25(docs_fts, 5.0, 1.0, 0.0) AS rank FROM docs_fts f - JOIN doc_shares ds ON ds.doc_id = f.doc_id AND ds.user_id = ? JOIN docs d ON d.id = f.doc_id LEFT JOIN folders fo ON fo.id = d.folder_id WHERE docs_fts MATCH ? AND f.project_id = ? + AND f.doc_id IN (SELECT doc_id FROM doc_shares WHERE project_id = ? AND user_id = ?) ORDER BY rank LIMIT 20 - `).bind(user.userId, ftsQuery, projectId).all(); + `).bind(ftsQuery, projectId, projectId, user.userId).all(); } else { docsPromise = env.DB.prepare(` SELECT f.doc_id, f.title, snippet(docs_fts, 1, '', '', '...', 24) AS excerpt, fo.name AS folder, d.updated_at, - bm25(docs_fts, 5.0, 1.0) AS rank + bm25(docs_fts, 5.0, 1.0, 0.0) AS rank FROM docs_fts f JOIN docs d ON d.id = f.doc_id LEFT JOIN folders fo ON fo.id = d.folder_id @@ -220,13 +223,13 @@ export async function handlePublicSearch( return okResponse({ docs, files: [], folders: [] } satisfies SearchResponse); } - const ftsQuery = sanitizeFtsQuery(q!); + const ftsQuery = siteFtsMatch(project.id, q!); const results = await env.DB.prepare(` SELECT f.doc_id, f.title, snippet(docs_fts, 1, '', '', '...', 24) AS excerpt, fo.name AS folder, d.updated_at, - bm25(docs_fts, 5.0, 1.0) AS rank + bm25(docs_fts, 5.0, 1.0, 0.0) AS rank FROM docs_fts f JOIN docs d ON d.id = f.doc_id LEFT JOIN folders fo ON fo.id = d.folder_id From 9780b7a945b2410a331156c22088444e12df664b Mon Sep 17 00:00:00 2001 From: Michael Burr Date: Mon, 2 Nov 2026 08:15:36 +0000 Subject: [PATCH 2/2] fix(search): make the docs_fts repair runbook survive wrangler on Windows The repair script's documented --file/--command invocations don't work: a remote --file goes through D1's bulk-import API instead of the query path migrations use, and npx's .cmd shim on Windows cuts a multi-line --command at its first newline (a leading comment line then runs nothing). Document the tested form (strip full-line comments, join to one line), exercise exactly that form in the repair test, and add a CLAUDE.md note that docs_fts must only be written through the lib/fts.ts statement helpers. --- CLAUDE.md | 1 + packages/api/scripts/repair-docs-fts.sql | 13 ++++++++++++- packages/api/src/lib/fts.test.ts | 6 +++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9ec0101c..9d689687 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,7 @@ Authentication uses `Authorization: Bearer ` headers. JWTs are issued by th - **API DB** (`cubedocs-main`): projects, docs (+ `doc_ai_summaries` satellite; `docs.slug` is the doc's custom public-URL segment from the `slug:` frontmatter key - unique per project, normalized + synced by `docOps.syncDocSlug` on every save path incl. collab checkpoints and restores, resolved by `/public/docs/:site/:idOrSlug`), doc_revisions, folders, files, members, site_reports + user_reports (abuse reports: public "Report Site" button → `POST /public/projects/:id/report`, profile-card "Report user" → authenticated `POST /users/:id/report`; both triaged via the admin worker's `/api/reports/{sites,users}` routes + the admin "Reports" page's Sites/Users tabs) - see `packages/api/migrations/`. Bound as `DB` on the API worker. - **Recycle bin** (API DB, `0062_add_trash.sql`, `lib/trash.ts`, `routes/trash.ts`, `components/RecycleBin.tsx` - shown as the "Recycle bin" section of Site Settings, `/projects/:id/trash` redirects there): deleting a doc/file/folder (app, or `/v1` doc DELETE) **moves its rows out of** `docs`/`files`/`folders` into `trash_items` JSON snapshots (one `trash_entries` row per user-visible entry; a folder bins its whole subtree as one entry, except the home doc, which is moved to the root). Because binned rows are physically gone from the live tables, **no reader needs a "not deleted" filter - don't add one, and don't hard-delete docs/files/folders from a route; call `moveToTrash`**. Cascade-lost satellites that aren't derivable (`doc_shares`, the `docs_fts` row, `site_reports.doc_id`) ride in `extra_json` and come back on restore; R2 objects + `asset_revisions` stay until purge. Restore/list = editor+, delete-forever/empty = admin+, auto-purge after 30 days on the daily cron. Site deletion (api + admin) must include `trash_items` ids in its R2 sweep. `trash.test.ts` runs against real SQLite via `src/test/sqliteD1.ts` (all migrations applied) - use that harness for SQL-heavy logic. +- **Doc search index** (API DB, `0064_rebuild_docs_fts.sql`, `lib/fts.ts`): `docs_fts` is FTS5 with an indexed `pid` = `hex(project_id)` token column plus a `docs_fts_map(rid, doc_id)` rowid map, so per-doc writes are rowid lookups and search MATCHes `pid:""` (`siteFtsMatch`) instead of scanning every site. **Never write `docs_fts` with raw SQL (`WHERE doc_id = ?` / `WHERE project_id = ?` are full scans, and a bare INSERT gets no `pid` or map row) - use the `fts*Statements` helpers**, which keep exactly one map row per FTS row and compose into a caller's batch (the insert + map statements must stay adjacent: `last_insert_rowid()`). Search weights stay `bm25(docs_fts, 5.0, 1.0, 0.0)` (pid weighted 0). `scripts/repair-docs-fts.sql` reconciles rows written by pre-0064 code. - **Admin account deletion is always delayed** (auth `0034_add_user_deletions.sql`, admin `lib/userDeletion.ts`, Users page checkboxes → bulk Delete/Disable/Re-enable/Cancel deletion): "Delete" only **schedules** - it disables the account now and writes a `user_deletions` row; the admin worker's hourly cron (`runDueUserDeletions`) performs the delete after 7 days, and any admin can cancel before then (restoring the prior moderation). Owned sites/orgs are deleted **only** when the operator ticked an explicit per-account box in the preview, scheduling refuses unless the confirmed ids exactly match current ownership (`sameIdSet`), and execution refuses if the account owns anything outside them. While a deletion is pending, `PATCH /api/users/:id` moderation is refused - cancel instead (the guard is inside the write, not just a pre-check). The cron claims each schedule by `(user_id, id)` with a lease; cancel is refused while leased or once `started_at` is set (first irreversible step), after which a failed run is retried to completion - a confirmed site/org that no longer exists counts as done. Real-SQLite scenario tests: `userDeletion.sqlite.test.ts`. **Don't add an immediate admin delete path.** Site deletion itself is shared via `lib/projectDelete.ts`. - **Auth DB** (`cubedocs-auth`): users, sessions, totp, webauthn_credentials, plus the 1:1 satellites `user_billing` (Stripe + plan state) and `user_preferences` (fonts, Ink cosmetics, timezone, bio, badges, opt-in profile `handle`) - see `packages/auth/migrations/`. The `handle` (`0035`, partial unique index, stored lowercase, rules in `packages/auth/src/handle.ts` mirrored by `frontend/src/lib/handle.ts`) powers the `/@` profile link: set/cleared via auth `/update-handle` (proxied as `PATCH /me/handle`), resolved by the API's `GET /users/by-handle/:handle` reading `AUTH_DB` directly. Null = link off. Admins moderate it via the admin worker's `PATCH /api/users/:id/handle` (Users → details → "Profile Link"; may assign reserved names) and can set `handle_locked`, which makes `/update-handle` refuse every user change (403 `reason: "locked"`, guarded inside the write too). Bound as `DB` on the auth worker (read+write), as `AUTH_DB` on the API worker (read-only by convention), and as `AUTH_DB` on the admin worker (read+write for moderation/badges/billing-grants/audit - see the verification-boundary note above). **A schema change to any column these direct readers/writers touch requires redeploying auth + api + admin (in that order).** diff --git a/packages/api/scripts/repair-docs-fts.sql b/packages/api/scripts/repair-docs-fts.sql index 0445115a..30888f1a 100644 --- a/packages/api/scripts/repair-docs-fts.sql +++ b/packages/api/scripts/repair-docs-fts.sql @@ -8,7 +8,18 @@ -- runs. Safe to run any number of times; on a consistent index it changes -- nothing. -- --- cd packages/api && npx wrangler d1 execute cubedocs-main --remote --file scripts/repair-docs-fts.sql +-- cd packages/api && npx wrangler d1 execute cubedocs-main --remote \ +-- --command="$(grep -v '^--' scripts/repair-docs-fts.sql | tr '\n' ' ')" +-- +-- (Git Bash/POSIX shell.) Why not plain `--file` / `--command "$(cat ...)"`: +-- a remote --file goes through D1's bulk-import API rather than the query +-- path `migrations apply` uses (the path that created and rebuilt docs_fts); +-- and on Windows `npx` runs through a .cmd shim that cuts an argument off at +-- its first newline, so a multi-line --command silently runs only its first +-- line. Stripping the full-line comments and joining the lines avoids both; +-- the output must show 8 result sets, the last two being the checks below. +-- Every step is defined on the current state alone, so a run interrupted +-- part-way is fixed by simply running it again. -- -- Old code always deletes EVERY row of a doc before inserting its new one, so -- if a doc has both a mapped row and unmapped rows, the mapped row was written diff --git a/packages/api/src/lib/fts.test.ts b/packages/api/src/lib/fts.test.ts index 6e0342c2..c528ad7b 100644 --- a/packages/api/src/lib/fts.test.ts +++ b/packages/api/src/lib/fts.test.ts @@ -261,7 +261,11 @@ describe("scripts/repair-docs-fts.sql", () => { const repair = readFileSync(fileURLToPath(new URL("../../scripts/repair-docs-fts.sql", import.meta.url)), "utf8"); const snapshot = () => all(sqlite, "SELECT doc_id, body, pid FROM docs_fts ORDER BY doc_id"); - sqlite.exec(repair); + // First run exactly as the runbook in the script's header feeds it to + // `wrangler d1 execute --command`: full-line comments stripped, one line. + const runbookForm = repair.split(/\r?\n/).filter(l => !l.startsWith("--")).join(" "); + expect(runbookForm).not.toContain("--"); + sqlite.exec(runbookForm); const once = snapshot(); expect(once).toEqual([ { doc_id: "a1", body: "mapped current", pid: ftsSiteToken(P1) },