diff --git a/.tmp/proof/document-index-retry.txt b/.tmp/proof/document-index-retry.txt new file mode 100644 index 0000000..98e60ae --- /dev/null +++ b/.tmp/proof/document-index-retry.txt @@ -0,0 +1,124 @@ +$ env -i HOME="$HOME" PATH="$PATH" UV_PYTHON=3.13 UV_FROZEN=1 MISE_YES=1 SLACK_BOT_TOKEN=xoxb-fixture-only SLACK_SIGNING_SECRET=fixture-only DATABASE_URL=sqlite:///.tmp-tests.db PYTEST_ADDOPTS=-rP mise run test-backend +tests/test_document_indexing.py::test_create_and_retry_keep_one_owned_chunk_and_preserve_saved_record PASSED [ 94%] +tests/test_document_indexing.py::test_retry_recovers_saved_document_after_embedding_failure_with_transcript PASSED [ 94%] +tests/test_document_indexing.py::test_retry_rechunks_and_removes_only_obsolete_owned_chunks PASSED [ 95%] +tests/test_document_indexing.py::test_retry_reads_latest_saved_content_title_and_source PASSED [ 95%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[lookup] PASSED [ 96%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[embed] PASSED [ 96%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[upsert] PASSED [ 97%] +tests/test_document_indexing.py::test_cleanup_failure_is_reported_and_retry_finishes_without_duplicates PASSED [ 97%] +tests/test_document_indexing.py::test_missing_saved_document_does_not_initialise_vector_store PASSED [ 98%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content- \n\t ] PASSED [ 98%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content-None] PASSED [ 99%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[title-] PASSED [ 99%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[source-None] PASSED [100%] +====================== 209 passed, 928 warnings in 6.48s ======================= + +Captured stdout from test_retry_recovers_saved_document_after_embedding_failure_with_transcript +{ + "query": "How do I reset VPN?", + "document_id": 1, + "top_k": 5, + "before_retry": [], + "after_two_retries": [ + { + "content": "Reset VPN via IT portal.", + "metadata": { + "chunk": 0, + "document_id": 1, + "source": "api", + "title": "VPN guide" + }, + "distance": 1.0 + } + ], + "chunk_count": 1, + "note": "Fixed-vector squared L2 distances prove recovery and identity, not production relevance." +} + +$ mise run dev-backend (with a local SQLite URL, fixed Chroma embeddings and a Slack-denying sitecustomize hook) +INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) +INFO: Application startup complete. +INFO: Application shutdown complete. + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/health +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:52 GMT +server: uvicorn +content-length: 20 +content-type: application/json + +{"status":"healthy"} + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/api/retrieval/preview -H 'Content-Type: application/json' -d '{"question": "How do I reset VPN?"}' +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:52 GMT +server: uvicorn +content-length: 93 +content-type: application/json + +{"question":"How do I reset VPN?","channel_id":null,"top_k":5,"max_distance":null,"items":[]} + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/api/documents -H 'Content-Type: application/json' -d '{"title": "VPN guide", "content": "Reset VPN via IT portal."}' +HTTP/1.1 503 Service Unavailable +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 83 +content-type: application/json + +{"detail":{"message":"Document saved but indexing is unavailable","document_id":1}} + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/api/documents/1 +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 164 +content-type: application/json + +{"id":1,"title":"VPN guide","content":"Reset VPN via IT portal.","source":"api","created_at":"2026-11-15T02:22:53.268264","updated_at":"2026-11-15T02:22:53.268271"} + +$ curl -sS --max-time 10 -i -X POST http://127.0.0.1:8000/api/documents/1/index +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 50 +content-type: application/json + +{"id":1,"message":"Document indexed successfully"} + +$ curl -sS --max-time 10 -i -X POST http://127.0.0.1:8000/api/documents/1/index +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 50 +content-type: application/json + +{"id":1,"message":"Document indexed successfully"} + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/api/retrieval/preview -H 'Content-Type: application/json' -d '{"question": "How do I reset VPN?"}' +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 220 +content-type: application/json + +{"question":"How do I reset VPN?","channel_id":null,"top_k":5,"max_distance":null,"items":[{"content":"Reset VPN via IT portal.","metadata":{"chunk":0,"document_id":1,"source":"api","title":"VPN guide"},"distance":1.0}]} + +$ curl -sS --max-time 10 -i http://127.0.0.1:8000/api/documents +HTTP/1.1 200 OK +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 174 +content-type: application/json + +{"items":[{"id":1,"title":"VPN guide","source":"api","created_at":"2026-11-15T02:22:53.268264","content_preview":"Reset VPN via IT portal."}],"total":1,"limit":50,"offset":0} + +$ curl -sS --max-time 10 -i -X POST http://127.0.0.1:8000/api/documents/999/index +HTTP/1.1 404 Not Found +date: Sun, 15 Nov 2026 02:22:53 GMT +server: uvicorn +content-length: 31 +content-type: application/json + +{"detail":"Document not found"} + diff --git a/backend/app/main.py b/backend/app/main.py index d4e71d8..62aa9be 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -6,7 +6,7 @@ from fastapi import Depends, FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel, StringConstraints, field_validator +from pydantic import BaseModel, StringConstraints, ValidationError, field_validator from sqlalchemy import func, or_ from sqlalchemy.orm import Session from starlette.concurrency import run_in_threadpool @@ -260,18 +260,29 @@ async def add_document(doc: DocumentRequest, db: Session = Depends(get_db)): db.add(db_doc) db.commit() db.refresh(db_doc) - - # Index in vector store - rag_engine.index_document(doc.title, doc.content, doc.source) - - return { - "id": db_doc.id, - "message": "Document added successfully" - } + except Exception as e: + db.rollback() logger.error(f"Error adding document: {e}") raise HTTPException(status_code=500, detail=str(e)) + await _index_saved_document(db_doc.id, doc) + return {"id": db_doc.id, "message": "Document added successfully"} + + +async def _index_saved_document(document_id: int, doc: DocumentRequest): + try: + await run_in_threadpool( + rag_engine.index_document, doc.title, doc.content, doc.source, + document_id=document_id, + ) + except Exception: + logger.exception("Indexing saved document %s failed", document_id) + raise HTTPException(status_code=503, detail={ + "message": "Document saved but indexing is unavailable", + "document_id": document_id, + }) + @app.get("/api/documents", response_model=DocumentListResponse) async def list_documents( @@ -327,6 +338,22 @@ async def get_document(document_id: int, db: Session = Depends(get_db)): return doc +@app.post("/api/documents/{document_id}/index") +async def index_saved_document(document_id: int, db: Session = Depends(get_db)): + """Retry indexing saved content without creating another document record.""" + doc = db.query(Document).filter(Document.id == document_id).first() + if doc is None: + raise HTTPException(status_code=404, detail="Document not found") + try: + validated = DocumentRequest(title=doc.title, content=doc.content, source=doc.source) + except ValidationError: + raise HTTPException( + status_code=422, detail="Saved document has invalid title, content or source", + ) + await _index_saved_document(doc.id, validated) + return {"id": doc.id, "message": "Document indexed successfully"} + + @app.get("/api/messages", response_model=List[MessageResponse]) async def list_messages( limit: int = 100, diff --git a/backend/app/rag.py b/backend/app/rag.py index d37a2e2..cd3bab8 100644 --- a/backend/app/rag.py +++ b/backend/app/rag.py @@ -204,7 +204,8 @@ def index_slack_thread(self, messages: List[Dict[str, Any]], channel_id: str): ids=list(chunks), ) - def index_document(self, title: str, content: str, source: str = "docs"): + def index_document(self, title: str, content: str, source: str = "docs", *, + document_id: int | None = None): """ Index a document for retrieval @@ -212,6 +213,7 @@ def index_document(self, title: str, content: str, source: str = "docs"): title: Document title content: Document content source: Source of the document + document_id: Saved record ID for retry-safe replacement of its chunks """ # Split content into chunks if needed chunks = self._split_text(content) @@ -225,7 +227,10 @@ def index_document(self, title: str, content: str, source: str = "docs"): for i in range(len(chunks)) ] - vector_store.add_documents(chunks, metadatas) + if document_id is None: + vector_store.add_documents(chunks, metadatas) + else: + vector_store.replace_document(document_id, chunks, metadatas) def _split_text(self, text: str) -> List[str]: """Split into overlapping character windows, stopping at the final chunk.""" diff --git a/backend/app/vector_store.py b/backend/app/vector_store.py index 55debb3..0c3d382 100644 --- a/backend/app/vector_store.py +++ b/backend/app/vector_store.py @@ -68,6 +68,28 @@ def add_documents(self, texts: List[str], metadatas: List[Dict[str, Any]] = None options = {"ids": ids} if ids is not None else {} return self.vector_store.add_texts(texts=texts, metadatas=metadatas, **options) + + def replace_document(self, document_id: int, texts: List[str], + metadatas: List[Dict[str, Any]]): + """Upsert owned chunks before removing obsolete chunks for a saved document. + + Existing chunks are not deleted before the upsert succeeds. Cleanup + failures propagate so the caller can retry the same stable IDs. + Unowned legacy chunks and other documents are never removed. + """ + if not texts: + raise ValueError("A document must contain at least one chunk") + self._ensure_initialized() + if self.vector_store is None: + raise RuntimeError("Vector store not initialized") + + previous = self.vector_store.get(where={"document_id": document_id}, include=[]) + ids = [f"document:{document_id}:{i}" for i in range(len(texts))] + owned_metadata = [{**metadata, "document_id": document_id} for metadata in metadatas] + self.add_documents(texts, owned_metadata, ids=ids) + obsolete = sorted(set(previous["ids"]) - set(ids)) + if obsolete: + self.vector_store.delete(ids=obsolete) def similarity_search(self, query: str, k: int = None, *, channel_id: str = None) -> List[Dict[str, Any]]: diff --git a/backend/tests/test_document_indexing.py b/backend/tests/test_document_indexing.py new file mode 100644 index 0000000..03e7122 --- /dev/null +++ b/backend/tests/test_document_indexing.py @@ -0,0 +1,249 @@ +"""Saved-document indexing with SQLite, local Chroma and fixed embeddings.""" +import json +import os +from unittest.mock import Mock, patch + +os.environ.setdefault("SLACK_BOT_TOKEN", "xoxb-fixture-only") +os.environ.setdefault("SLACK_SIGNING_SECRET", "fixture-only") + +import chromadb +import pytest +from chromadb.config import Settings as ChromaSettings +from chromadb.telemetry.product.posthog import Posthog +from fastapi.testclient import TestClient +from langchain_community.vectorstores import Chroma +from langchain_core.embeddings import Embeddings +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.config import settings +from app.database import Base, Document, get_db +from app.vector_store import VectorStore +import app.rag as rag_module + +with patch("slack_sdk.WebClient.auth_test", return_value={ + "ok": True, "bot_id": "BTEST", "user_id": "UTEST", "team_id": "TTEST", +}): + from app.main import app + + +CONTENT = "Reset VPN via IT portal." +QUESTION = "How do I reset VPN?" + + +class DocumentEmbeddings(Embeddings): + def embed_documents(self, texts): + # The full guide has distance 1, shorter fragments have larger distances. + return [[float(len(CONTENT) - len(text) + 1), 1.0] for text in texts] + + def embed_query(self, text): + return [0.0, 1.0] + + +@pytest.fixture +def document_api(tmp_path, monkeypatch): + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool, + ) + sessions = sessionmaker(bind=engine) + Base.metadata.create_all(engine) + + def get_test_db(): + with sessions() as db: + yield db + + previous = app.dependency_overrides.get(get_db) + app.dependency_overrides[get_db] = get_test_db + monkeypatch.setattr(Posthog, "capture", lambda *args, **kwargs: None) + monkeypatch.setattr(settings, "RETRIEVAL_TOP_K", 5) + monkeypatch.setattr(settings, "RETRIEVAL_MAX_DISTANCE", None) + monkeypatch.setattr(settings, "CHUNK_SIZE", 1000) + monkeypatch.setattr(settings, "CHUNK_OVERLAP", 0) + vector = VectorStore() + vector.vector_store = Chroma( + client=chromadb.PersistentClient( + path=str(tmp_path), settings=ChromaSettings(anonymized_telemetry=False), + ), + collection_name="saved-document-fixture", embedding_function=DocumentEmbeddings(), + ) + vector._initialized = True + monkeypatch.setattr(rag_module, "vector_store", vector) + blocked = Mock(side_effect=AssertionError("Document indexing must not contact Slack or generate")) + monkeypatch.setattr("slack_sdk.WebClient.api_call", blocked) + monkeypatch.setattr("slack_bolt.adapter.socket_mode.SocketModeHandler.start", blocked) + monkeypatch.setattr(rag_module.agent_runtime, "generate", blocked) + monkeypatch.setattr(rag_module.agent_runtime, "submit", blocked) + try: + yield TestClient(app, raise_server_exceptions=False), sessions, vector + blocked.assert_not_called() + finally: + if previous is None: + app.dependency_overrides.pop(get_db, None) + else: + app.dependency_overrides[get_db] = previous + engine.dispose() + + +def create_document(client, content=CONTENT): + return client.post("/api/documents", json={"title": "VPN guide", "content": content}) + + +def retry_index(client, document_id): + return client.post(f"/api/documents/{document_id}/index") + + +def test_create_and_retry_keep_one_owned_chunk_and_preserve_saved_record(document_api): + client, sessions, vector = document_api + response = create_document(client) + assert response.status_code == 200 + document_id = response.json()["id"] + original = client.get(f"/api/documents/{document_id}").json() + for _ in range(2): + response = retry_index(client, document_id) + assert response.status_code == 200 + assert response.json() == {"id": document_id, "message": "Document indexed successfully"} + assert vector.vector_store.get()["ids"] == [f"document:{document_id}:0"] + assert vector.similarity_search(QUESTION) == [{ + "content": CONTENT, "distance": 1.0, + "metadata": {"title": "VPN guide", "source": "api", "chunk": 0, "document_id": document_id}, + }] + assert client.get(f"/api/documents/{document_id}").json() == original + with sessions() as db: + assert db.query(Document).count() == 1 + + +def test_retry_recovers_saved_document_after_embedding_failure_with_transcript(document_api): + client, sessions, vector = document_api + with patch.object(vector.vector_store._embedding_function, "embed_documents", + side_effect=RuntimeError("Private provider credentials")): + failed = create_document(client) + assert failed.status_code == 503 + document_id = failed.json()["detail"]["document_id"] + assert failed.json() == {"detail": { + "message": "Document saved but indexing is unavailable", "document_id": document_id, + }} + assert "Private" not in failed.text + assert client.get(f"/api/documents/{document_id}").json()["content"] == CONTENT + before = vector.similarity_search(QUESTION) + assert before == [] + for _ in range(2): + assert retry_index(client, document_id).status_code == 200 + after = vector.similarity_search(QUESTION) + assert [item["content"] for item in after] == [CONTENT] + assert [item["distance"] for item in after] == [1.0] + with sessions() as db: + assert db.query(Document).count() == 1 + print(json.dumps({ + "query": QUESTION, "document_id": document_id, "top_k": 5, + "before_retry": before, "after_two_retries": after, + "chunk_count": vector.vector_store._collection.count(), + "note": "Fixed-vector squared L2 distances prove recovery and identity, not production relevance.", + }, indent=2)) + + +def test_retry_rechunks_and_removes_only_obsolete_owned_chunks(document_api, monkeypatch): + client, _, vector = document_api + monkeypatch.setattr(settings, "CHUNK_SIZE", 8) + document_id = create_document(client).json()["id"] + assert len(vector.vector_store.get(where={"document_id": document_id})["ids"]) == 3 + other_id = create_document(client).json()["id"] # Same title and source, distinct owner. + legacy_ids = vector.add_documents([CONTENT], [{"title": "VPN guide", "source": "api"}]) + slack_ids = vector.add_documents([CONTENT], [{"source": "slack", "channel_id": "CTEST"}]) + other_before = vector.vector_store.get(where={"document_id": other_id}) + monkeypatch.setattr(settings, "CHUNK_SIZE", 1000) + for _ in range(2): + assert retry_index(client, document_id).status_code == 200 + owned = vector.vector_store.get(where={"document_id": document_id}) + assert owned["ids"] == [f"document:{document_id}:0"] + assert owned["documents"] == [CONTENT] + assert vector.vector_store.get(where={"document_id": other_id}) == other_before + assert set(vector.vector_store.get()["ids"]) == { + *owned["ids"], *other_before["ids"], *legacy_ids, *slack_ids, + } + + +def test_retry_reads_latest_saved_content_title_and_source(document_api): + client, sessions, vector = document_api + document_id = create_document(client).json()["id"] + updated_content = "Ask IT for access." + with sessions() as db: + db.query(Document).filter(Document.id == document_id).update({ + "title": "VPN handbook", "content": updated_content, "source": "docs", + }) + db.commit() + assert retry_index(client, document_id).status_code == 200 + assert vector.vector_store.get()["ids"] == [f"document:{document_id}:0"] + assert vector.similarity_search(QUESTION) == [{ + "content": updated_content, + "distance": float(len(CONTENT) - len(updated_content) + 1) ** 2, + "metadata": {"title": "VPN handbook", "source": "docs", "chunk": 0, "document_id": document_id}, + }] + + +@pytest.mark.parametrize("stage", ["lookup", "embed", "upsert"]) +def test_retry_failure_keeps_previous_chunks_and_saved_content(document_api, monkeypatch, stage): + client, _, vector = document_api + monkeypatch.setattr(settings, "CHUNK_SIZE", 8) + document_id = create_document(client).json()["id"] + before = vector.vector_store.get(include=["documents", "metadatas", "embeddings"]) + saved = client.get(f"/api/documents/{document_id}").json() + monkeypatch.setattr(settings, "CHUNK_SIZE", 1000) + if stage == "lookup": + target, method = vector.vector_store, "get" + elif stage == "embed": + target, method = vector.vector_store._embedding_function, "embed_documents" + else: + target, method = vector.vector_store._collection, "upsert" + with patch.object(target, method, side_effect=RuntimeError("Private provider details")): + failed = retry_index(client, document_id) + assert failed.status_code == 503 + assert "Private" not in failed.text + after = vector.vector_store.get(include=["documents", "metadatas", "embeddings"]) + assert after["documents"] == before["documents"] + assert after["metadatas"] == before["metadatas"] + assert after["embeddings"].tolist() == before["embeddings"].tolist() + assert client.get(f"/api/documents/{document_id}").json() == saved + assert retry_index(client, document_id).status_code == 200 + assert vector.vector_store.get()["documents"] == [CONTENT] + + +def test_cleanup_failure_is_reported_and_retry_finishes_without_duplicates(document_api, monkeypatch): + client, _, vector = document_api + monkeypatch.setattr(settings, "CHUNK_SIZE", 8) + document_id = create_document(client).json()["id"] + monkeypatch.setattr(settings, "CHUNK_SIZE", 1000) + with patch.object(vector.vector_store, "delete", side_effect=RuntimeError("Cleanup unavailable")): + assert retry_index(client, document_id).status_code == 503 + assert len(vector.vector_store.get()["ids"]) == 3 + assert retry_index(client, document_id).status_code == 200 + assert vector.vector_store.get()["ids"] == [f"document:{document_id}:0"] + assert vector.vector_store.get()["documents"] == [CONTENT] + + +def test_missing_saved_document_does_not_initialise_vector_store(document_api, monkeypatch): + client, _, vector = document_api + index = Mock(side_effect=AssertionError("Missing document must not be indexed")) + monkeypatch.setattr(vector, "replace_document", index) + response = retry_index(client, 999) + assert response.status_code == 404 + assert response.json() == {"detail": "Document not found"} + index.assert_not_called() + + +@pytest.mark.parametrize("field,value", [ + ("content", " \n\t "), ("content", None), ("title", ""), ("source", None), +]) +def test_invalid_legacy_document_is_rejected_without_changing_chunks(document_api, monkeypatch, field, value): + client, sessions, vector = document_api + document_id = create_document(client).json()["id"] + with sessions() as db: + db.query(Document).filter(Document.id == document_id).update({field: value}) + db.commit() + index = Mock(side_effect=AssertionError("Invalid document must not be indexed")) + monkeypatch.setattr(vector, "replace_document", index) + response = retry_index(client, document_id) + assert response.status_code == 422 + assert response.json() == {"detail": "Saved document has invalid title, content or source"} + index.assert_not_called() + assert vector.vector_store.get()["documents"] == [CONTENT] diff --git a/backend/tests/test_documents.py b/backend/tests/test_documents.py index cc707dc..d077a71 100644 --- a/backend/tests/test_documents.py +++ b/backend/tests/test_documents.py @@ -251,8 +251,8 @@ def test_creates_readable_document_and_indexes_preserved_content(self, source): with patch.object(rag_engine, "index_document") as index: response = client.post("/api/documents", json=payload) assert response.status_code == 200 - index.assert_called_once_with("VPN guide", content, expected_source) document_id = response.json()["id"] + index.assert_called_once_with("VPN guide", content, expected_source, document_id=document_id) detail = client.get(f"/api/documents/{document_id}").json() assert detail["title"] == "VPN guide" assert detail["content"] == content diff --git a/docs/retrieval.md b/docs/retrieval.md index 27f0512..f8b4341 100644 --- a/docs/retrieval.md +++ b/docs/retrieval.md @@ -9,7 +9,9 @@ positive, and `CHUNK_OVERLAP` must be non-negative and smaller than `CHUNK_SIZE` Invalid settings are rejected at startup, before indexing can begin. Splitting stops when a chunk reaches the end of the content; it does not index redundant trailing overlap fragments. Existing stored fragments are not removed by this -change. Rebuild the collection from its original sources to remove them. +change. Saved document retries described below replace chunks carrying their +document ID. Unowned historical chunks need a collection rebuild from their +original sources. Set `RETRIEVAL_MAX_DISTANCE` to exclude weak matches from the retrieved context. Leave it unset to keep all top-k matches, which preserves the default behaviour. @@ -85,6 +87,44 @@ of `CTARGET` and a distance cutoff of 1. It contains no real workspace content. The [narrow screen capture](images/retrieval-preview-mobile.png) shows the same result at a width of 390 pixels. +## Retry saved document indexing + +`POST /api/documents/{document_id}/index` indexes the saved title, content and +source without creating or modifying a database record. Find the ID with +`GET /api/documents`. A missing record returns HTTP 404; invalid legacy title, +content or source fields return HTTP 422 before indexing starts. + +```sh +curl -sS -X POST http://localhost:8000/api/documents/1/index +``` + +Success returns HTTP 200 with `id` and `message`. Indexing failures return HTTP +503 with `detail.message` and `detail.document_id`, so the same record can be +retried after the cause is fixed. `POST /api/documents` uses the same indexing +path. If saving succeeds but indexing fails, it now returns this HTTP 503 shape +with the saved ID. Retry the index endpoint instead of posting the document +again, which would create a separate record. + +API-created and retried documents use stable IDs for each chunk and include +`document_id` in retrieval metadata. Repeated indexing replaces their chunks. +Changing chunk settings and retrying removes obsolete owned chunks after the +new chunks are stored successfully. Embedding failures leave existing chunks +available; a cleanup failure may leave a mix of old and new chunks until a +successful retry. Database storage and vector indexing are separate operations, +not an atomic transaction. + +Chunks belonging to other document IDs, Slack messages, and legacy chunks +without a document ID are left alone. Retrying a legacy document can therefore +leave historical duplicates. Titles and sources are not unique identities, so +automatic legacy cleanup would risk deleting another document. File indexing +without a saved document ID retains its existing behaviour. + +This endpoint calls the configured embedding provider, but does not fetch from +Slack or generate an answer. Keep it behind the same trusted access controls as +the other management endpoints. The offline tests use temporary local Chroma +and fixed embeddings; the recovery transcript is included in the backend gate +output when run with `PYTEST_ADDOPTS=-rP`. + ## Saved Slack threads Slack thread indexing uses a stable chunk ID built from the channel ID and diff --git a/mise.toml b/mise.toml index ebc3c19..a353d8f 100644 --- a/mise.toml +++ b/mise.toml @@ -51,7 +51,7 @@ run = "docker compose ps" [tasks.test-backend] description = "Run backend tests" dir = "backend" -run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py -v" +run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py tests/test_document_indexing.py -v" [tasks.migrate] description = "Run database migrations (upgrade to latest)"