CLEAN-100: bound every LightRAG call, say so when it hangs

api/src/slices/reins/README.md

@@ -118,6 +118,26 @@ LightRAG's `updated_at` on the failed verdict travels with the row when it is
 re-queued (`indexRequeuedOverAt`); seeing that same verdict again is "not
 reached yet", any other timestamp is a new failure.
 
+## LightRAG can be up and answer nothing
+
+On 2026-09-18 LightRAG's Postgres pod was moved to another node on the Mazda
+dev cluster. The data survived (PVC) and Postgres was back within a minute, but
+LightRAG kept its pool of connections to the pod that was gone: `/health` and
+`/documents/pipeline_status` answered in half a second, everything that touches
+the database hung, and its log went silent. It does not recover by itself;
+deleting the LightRAG pod is the fix. Agents saw only the MCP client's own
+60 s limit (`MCP error -32001: Request timed out`), which sent people looking
+at MCP.
+
+Every call in `LightragHttpClient` is therefore bounded and a cut-short one is
+a `LightragTimeoutError`: 30 s for status reads, 50 s for a query (under the
+MCP client's 60 s, so `query_knowledge` answers in words), two minutes for an
+upload, five for a delete. When the document listing times out, the source
+gateway stops talking to that instance for the run or pass (each upload would
+only run into its own limit, one after another), leaves every row as it was,
+and logs one error line that names the state: "answers /health but /documents
+timed out ... restart the LightRAG pod". That line is the one to alert on.
+
 ## Changing the extraction model requires clearing the LLM cache
 
 LightRAG caches every extraction and summary response in `lightrag_llm_cache`,

api/src/slices/reins/knowledge/knowledge.tool.spec.ts

@@ -11,6 +11,7 @@ import { IKnowledgeGateway } from './domain/knowledge.gateway';
 import { IAgentGateway } from '#/agent/agent/domain';
 import { ITemplateGateway } from '#/agent/template/domain';
 import { IAuthTokenPayload } from '#/user/auth/domain';
+import { LightragTimeoutError } from '../lightrag/domain/lightrag.types';
 
 const K2_SECRET = 'the Grangemouth override code is 9944';
 
@@ -28,12 +29,13 @@ interface Harness {
   findExistingByIds: jest.Mock;
 }
 
-function makeHarness(boundIds: string[]): Harness {
+function makeHarness(boundIds: string[], failWith?: Error): Harness {
   const queriedIds: string[] = [];
 
   const knowledgeService = {
     query: jest.fn(async (knowledgeId: string) => {
       queriedIds.push(knowledgeId);
+      if (failWith) throw failWith;
       if (knowledgeId === 'k2') {
         // If the tool ever lets a query through to K2, the secret leaks
         // into the result and the assertions below catch it.
@@ -338,3 +340,40 @@ describe('FR-006 — a multi-base answer attributes each part', () => {
     expect(parsed.answer).toBe('shared answer');
   });
 });
+
+describe('query_knowledge when the knowledge base does not answer', () => {
+  test('says so in words and asks the agent to come back later', async () => {
+    // 2026-09-18: LightRAG hung on dead database connections and the agent
+    // only ever saw the MCP client's "-32001: Request timed out", which sent
+    // everyone looking at MCP.
+    const { tool } = makeHarness(
+      ['k1'],
+      new LightragTimeoutError('/query', 50_000),
+    );
+
+    const result = await tool.query(
+      { query: 'CX-5 advantages over RAV4' },
+      null,
+      agentRequest('agent-1'),
+    );
+
+    expect(result.isError).toBeUndefined();
+    expect(textOf(result)).toContain('Relays is not answering right now');
+    expect(textOf(result)).toContain('LightRAG /query timed out after 50 s');
+    expect(textOf(result)).toContain('try again in a few minutes');
+  });
+
+  test('keeps the plain wording for any other failure', async () => {
+    const { tool } = makeHarness(['k1'], new Error('LightRAG /query failed: 502'));
+
+    const result = await tool.query(
+      { query: 'anything' },
+      null,
+      agentRequest('agent-1'),
+    );
+
+    expect(textOf(result)).toContain(
+      'Relays could not be reached: LightRAG /query failed: 502',
+    );
+  });
+});

api/src/slices/reins/knowledge/knowledge.tool.ts

@@ -8,6 +8,7 @@ import { ITemplateGateway } from '#/agent/template/domain';
 import { IDynamicallyDescribedTool } from '#/mcp/interfaces/dynamic-description.interface';
 import { KnowledgeService } from './domain/knowledge.service';
 import { IKnowledgeGateway } from './domain/knowledge.gateway';
+import { LightragTimeoutError } from '../lightrag/domain/lightrag.types';
 
 // The batching sentence is not stylistic advice: one call spends several
 // seconds inside the knowledge service composing an answer, and the service
@@ -148,10 +149,19 @@ export class KnowledgeTool implements IDynamicallyDescribedTool {
             return { knowledge_id: id, knowledge_name, ...r };
           } catch (e) {
             const message = e instanceof Error ? e.message : 'query failed';
+            // Logged here because this failure never reaches the outer catch:
+            // it is folded into a normal result, and on 2026-09-18 that left
+            // an hour of timeouts with no line of ours in the API log.
+            this.logger.warn(
+              `query_knowledge failed for agent=${callerAgentId} knowledge=${id}: ${message}`,
+            );
             return {
               knowledge_id: id,
               knowledge_name,
-              error: `Knowledge base ${knowledge_name ?? id} could not be reached: ${message}`,
+              error:
+                e instanceof LightragTimeoutError
+                  ? `Knowledge base ${knowledge_name ?? id} is not answering right now (${message}). It is most likely restarting: say so to the user and try again in a few minutes instead of repeating the call straight away.`
+                  : `Knowledge base ${knowledge_name ?? id} could not be reached: ${message}`,
             };
           }
         }),

api/src/slices/reins/lightrag/data/lightragHttp.client.spec.ts

@@ -0,0 +1,134 @@
+import { LightragHttpClient } from './lightragHttp.client';
+import { LightragTimeoutError } from '../domain/lightrag.types';
+
+type FetchImpl = typeof fetch;
+
+type FetchMock = jest.Mock<Promise<Response>, [string, RequestInit?]>;
+
+/** A LightRAG that takes the connection and never answers, until aborted. */
+function hangingFetch(): FetchMock {
+  return jest.fn(
+    (_url: string, init?: RequestInit) =>
+      new Promise<Response>((_resolve, reject) => {
+        init?.signal?.addEventListener('abort', () => {
+          reject(
+            Object.assign(new Error('This operation was aborted'), {
+              name: 'AbortError',
+            }),
+          );
+        });
+      }),
+  );
+}
+
+function makeClient(fetchImpl: FetchMock): LightragHttpClient {
+  return new LightragHttpClient({
+    resolveConfig: () =>
+      Promise.resolve({ url: 'http://lightrag:9621', apiKey: 'k', enabled: true }),
+    fetchImpl: fetchImpl as unknown as FetchImpl,
+  });
+}
+
+describe('LightragHttpClient: a LightRAG that never answers', () => {
+  beforeEach(() => {
+    jest.useFakeTimers();
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  it('gives up on a query before the MCP client does, and says what it waited for', async () => {
+    // The MCP client stops at 60 s with a transport error that reads as "MCP
+    // is broken". Stopping first is what lets the tool answer in words.
+    const client = makeClient(hangingFetch());
+
+    const pending = client.query({ knowledgeId: 'k1', query: 'anything' });
+    const outcome = expect(pending).rejects.toThrow(
+      'LightRAG /query timed out after 50 s',
+    );
+    await jest.advanceTimersByTimeAsync(50_000);
+
+    await outcome;
+    await expect(pending).rejects.toBeInstanceOf(LightragTimeoutError);
+  });
+
+  it('does not give up on a query a second early', async () => {
+    const client = makeClient(hangingFetch());
+    let settled = false;
+
+    const pending = client
+      .query({ knowledgeId: 'k1', query: 'anything' })
+      .catch(() => undefined)
+      .finally(() => {
+        settled = true;
+      });
+    await jest.advanceTimersByTimeAsync(49_000);
+    expect(settled).toBe(false);
+
+    await jest.advanceTimersByTimeAsync(1_000);
+    await pending;
+    expect(settled).toBe(true);
+  });
+
+  it('bounds the status reads the reconciler makes inside its lock', async () => {
+    // One unbounded listDocuments and no reconcile pass ever runs again.
+    const client = makeClient(hangingFetch());
+
+    const documents = expect(client.listDocuments('k1')).rejects.toThrow(
+      'LightRAG /documents timed out after 30 s',
+    );
+    const pipeline = expect(client.getPipelineStatus('k1')).rejects.toThrow(
+      'LightRAG /documents/pipeline_status timed out after 30 s',
+    );
+    const track = expect(
+      client.getTrackStatus('k1', 'track-9'),
+    ).rejects.toBeInstanceOf(LightragTimeoutError);
+    await jest.advanceTimersByTimeAsync(30_000);
+
+    await Promise.all([documents, pipeline, track]);
+  });
+
+  it('hands the abort signal to fetch so the socket is released too', async () => {
+    const fetchImpl = hangingFetch();
+    const client = makeClient(fetchImpl);
+
+    const pending = client.listDocuments('k1').catch(() => undefined);
+    await jest.advanceTimersByTimeAsync(30_000);
+    await pending;
+
+    const [, init] = fetchImpl.mock.calls[0];
+    expect(init?.signal?.aborted).toBe(true);
+  });
+});
+
+describe('LightragHttpClient: a LightRAG that answers', () => {
+  it('returns the answer and leaves no timer behind', async () => {
+    jest.useFakeTimers();
+    try {
+      const fetchImpl: FetchMock = jest.fn((_url: string) =>
+        Promise.resolve(
+          new Response(JSON.stringify({ statuses: {} }), { status: 200 }),
+        ),
+      );
+      const client = makeClient(fetchImpl);
+
+      expect(await client.listDocuments('k1')).toEqual([]);
+      expect(jest.getTimerCount()).toBe(0);
+    } finally {
+      jest.useRealTimers();
+    }
+  });
+
+  it('passes any other failure through untouched', async () => {
+    const fetchImpl: FetchMock = jest.fn((_url: string) =>
+      Promise.reject<Response>(new TypeError('fetch failed')),
+    );
+    const client = makeClient(fetchImpl);
+
+    const failure = client.listDocuments('k1');
+
+    await expect(failure).rejects.toThrow('fetch failed');
+    await expect(failure).rejects.not.toBeInstanceOf(LightragTimeoutError);
+  });
+});