From 41618a5037341fb2ec0338e7a92c73eb232d993b Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 26 Sep 2026 17:35:31 -0700 Subject: [PATCH] fix(retention): purge deletes and forget redacts the raw capture Purge left the document's raw_capture rows and forget left raw_capture.raw_text, so deleted text stayed stored. Purge now deletes the raw captures only the document's versions referenced, in the same transaction; forget redacts them. --- README.md | 18 +++++----- docs/RETENTION.md | 8 +++-- e2e/grants.test.ts | 10 ++++++ src/services/retention.ts | 74 +++++++++++++++++++++++++++++++-------- 4 files changed, 84 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 599b336..2067184 100644 --- a/README.md +++ b/README.md @@ -75,15 +75,15 @@ Other root exports: `MemoryError` and `RerankConfigError`, the `SEARCH_LIMIT_*` Returns a `Hono` sub-app. Mount it at `/api/tenants/:tenantId/memory`, below the middleware that sets `principal` and `tenant` on the context (Interchange's `createResolveTenant` does). Identity always comes from the context, never from the request body. -| Route | Grant | Does | -| ------------------------------------------- | --------------- | ---------------------------------------------------------------- | -| `POST /add` | `memory:add` | Adds a document, or a new version of one. | -| `POST /search` | `memory:search` | Hybrid search over the documents the caller can see. | -| `GET /list` | `memory:search` | Recent documents the caller can see. | -| `GET /feed` | `memory:search` | New versions after a `feed_seq` cursor, for consumers. | -| `POST /documents/:documentId/forget` | `memory:forget` | Drops a document from search and redacts its text. Creator only. | -| `POST /documents/:documentId/purge` | `memory:purge` | Deletes a document and its versions. Creator only. | -| `POST /versions/:versionId/retention-class` | `memory:forget` | Sets a version's retention class. Creator only. | +| Route | Grant | Does | +| ------------------------------------------- | --------------- | ------------------------------------------------------------------------------------------- | +| `POST /add` | `memory:add` | Adds a document, or a new version of one. | +| `POST /search` | `memory:search` | Hybrid search over the documents the caller can see. | +| `GET /list` | `memory:search` | Recent documents the caller can see. | +| `GET /feed` | `memory:search` | New versions after a `feed_seq` cursor, for consumers. | +| `POST /documents/:documentId/forget` | `memory:forget` | Drops a document from search and redacts its text, including the raw capture. Creator only. | +| `POST /documents/:documentId/purge` | `memory:purge` | Deletes a document, its versions and its raw captures. Creator only. | +| `POST /versions/:versionId/retention-class` | `memory:forget` | Sets a version's retention class. Creator only. | A missing principal answers `401` and a missing grant answers `403`. `callerResolver` is optional: it resolves a non-session caller to a `{ tenantId, principalId }` before the same grant checks run, and a malformed result answers `500`. diff --git a/docs/RETENTION.md b/docs/RETENTION.md index 642b4ad..6f69347 100644 --- a/docs/RETENTION.md +++ b/docs/RETENTION.md @@ -44,11 +44,13 @@ Service module: `src/services/retention.ts`. **Tombstone vs. hard delete stay distinct verbs, distinct grants.** A UI offering "forget this" must never be one flag away from "shred this" by accident. `forget` (tombstone) is **not** an undo-able action: the document -stops appearing in search/feed and its chunk text is overwritten with -`[redacted]` — the original content does not survive, there is no +stops appearing in search/feed, its chunk text is overwritten with +`[redacted]`, and so is the text of any raw capture only its versions +referenced — the original content does not survive, there is no un-tombstone/restore verb, and only version metadata (status, timestamps, retention class) remains for audit. `purge` (hard delete) goes further and -removes the document row itself; it has its own grant action and is refused +removes the document row itself, along with the raw captures only its +versions referenced; it has its own grant action and is refused outright while a `durable`-class version on the document is untombstoned. The distinction that matters is _what's still queryable_: after `forget` a document row and its metadata still exist (for audit) but its content is diff --git a/e2e/grants.test.ts b/e2e/grants.test.ts index 2b4ce6b..9f9cbbb 100644 --- a/e2e/grants.test.ts +++ b/e2e/grants.test.ts @@ -96,6 +96,12 @@ describe.skipIf(testDatabaseUrl() === undefined)( const [row] = await db.sql<{ status: string }[]>` SELECT status FROM memory.version WHERE document_id = ${documentId}`; expect(row?.status).toBe("tombstoned"); + const raw = await db.sql<{ raw_text: string | null }[]>` + SELECT r.raw_text FROM memory.raw_capture r + JOIN memory.version v ON v.raw_capture_id = r.id + WHERE v.document_id = ${documentId}`; + expect(raw.length).toBeGreaterThan(0); + for (const r of raw) expect(r.raw_text).not.toContain("Forget me"); }); test("purge removes the document, its versions and its chunks", async () => { @@ -118,6 +124,10 @@ describe.skipIf(testDatabaseUrl() === undefined)( (SELECT count(*)::int FROM memory.version WHERE document_id = ${documentId}) AS versions, (SELECT count(*)::int FROM memory.chunk WHERE document_id = ${documentId}) AS chunks`; expect(left).toEqual({ docs: 0, versions: 0, chunks: 0 }); + const [raw] = await db.sql<{ n: number }[]>` + SELECT count(*)::int AS n FROM memory.raw_capture + WHERE raw_text LIKE '%Purge me%'`; + expect(raw?.n).toBe(0); }); test("legacy capture and find grants authorize neither add nor search", async () => { diff --git a/src/services/retention.ts b/src/services/retention.ts index c9290db..f248802 100644 --- a/src/services/retention.ts +++ b/src/services/retention.ts @@ -5,7 +5,12 @@ import { and, eq, inArray, isNotNull, lt, or, sql } from "drizzle-orm"; import type { Db } from "../db/client.js"; -import { memoryChunk, memoryDocument, memoryVersion } from "../db/schema.js"; +import { + memoryChunk, + memoryDocument, + memoryVersion, + rawCapture, +} from "../db/schema.js"; import type { RetentionClass } from "../core/enums.js"; export type RetentionMutationResult = { @@ -46,7 +51,8 @@ export async function deprecateVersion( } /** - * Tombstone: hide from search/feed, redact chunk text, keep row for audit. + * Tombstone: hide from search/feed, redact chunk and raw capture text, keep + * rows for audit. * Applies to the document's live active (or deprecated) versions. */ export async function tombstoneDocument( @@ -84,12 +90,30 @@ export async function tombstoneDocument( eq(memoryChunk.documentId, input.documentId), ), ); + await db + .update(rawCapture) + .set({ rawText: "[redacted]", rawBytes: null }) + .where( + inArray( + rawCapture.id, + db + .select({ id: memoryVersion.rawCaptureId }) + .from(memoryVersion) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + ), + ), + ), + ); } return { versions: versions.length }; } /** - * Hard-delete a document (cascade chunks/versions/edges via FKs where set). + * Hard-delete a document (cascade chunks/versions/edges via FKs where set) + * and the raw captures only its versions referenced. * Blocked for durable retention_class on any non-tombstoned version. */ export async function hardDeleteDocument( @@ -119,17 +143,39 @@ export async function hardDeleteDocument( }; } - const deleted = await db - .delete(memoryDocument) - .where( - and( - eq(memoryDocument.tenantId, input.tenantId), - eq(memoryDocument.id, input.documentId), - ), - ) - .returning({ id: memoryDocument.id }); - - return { deleted: deleted.length > 0 }; + return db.transaction(async (tx) => { + const captures = await tx + .selectDistinct({ id: memoryVersion.rawCaptureId }) + .from(memoryVersion) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + isNotNull(memoryVersion.rawCaptureId), + ), + ); + const deleted = await tx + .delete(memoryDocument) + .where( + and( + eq(memoryDocument.tenantId, input.tenantId), + eq(memoryDocument.id, input.documentId), + ), + ) + .returning({ id: memoryDocument.id }); + const captureIds = captures.flatMap((c) => (c.id === null ? [] : [c.id])); + if (captureIds.length > 0) { + await tx + .delete(rawCapture) + .where( + and( + inArray(rawCapture.id, captureIds), + sql`NOT EXISTS (SELECT 1 FROM ${memoryVersion} WHERE ${memoryVersion.rawCaptureId} = ${rawCapture.id})`, + ), + ); + } + return { deleted: deleted.length > 0 }; + }); } /**