feat: add chaos, SDK, formal verification, and Helm scaffolding for #…

.github/workflows/chaos-testing.yml

@@ -0,0 +1,66 @@
+# Chaos Engineering / fault-injection verification (issue #1477).
+#
+# Runs the resilience suite against a Toxiproxy fault-injection stack:
+#   - nightly (catches drift regressions while nobody is merging),
+#   - on release-candidate branches,
+#   - and on pull requests that touch the backend.
+name: Chaos Testing
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+  push:
+    branches:
+      - 'release/**'
+  pull_request:
+    branches: [main, develop]
+    paths:
+      - 'backend/**'
+      - '.github/workflows/chaos-testing.yml'
+
+jobs:
+  chaos:
+    name: Resilience chaos suite
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      TOXIPROXY_URL: http://127.0.0.1:8474
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+          cache-dependency-path: package-lock.json
+
+      - name: Install dependencies
+        run: npm ci --include=optional
+
+      - name: Start chaos stack (Toxiproxy + deps)
+        run: docker compose -f backend/tests/chaos/docker-compose.chaos.yml up -d --wait
+
+      - name: Show stack state
+        run: docker compose -f backend/tests/chaos/docker-compose.chaos.yml ps
+
+      # Coverage is disabled here on purpose: this job runs a single spec, so the
+      # global 60% coverage thresholds from vitest.config.ts can never be met.
+      # The repository-wide coverage gate runs in ci.yml.
+      - name: Run chaos resilience tests
+        run: npx vitest run tests/chaos/resilience.chaos.test.ts --coverage.enabled=false
+        working-directory: backend
+
+      - name: Dump chaos stack logs on failure
+        if: failure()
+        run: docker compose -f backend/tests/chaos/docker-compose.chaos.yml logs --no-color
+
+      - name: Tear down chaos stack
+        if: always()
+        run: docker compose -f backend/tests/chaos/docker-compose.chaos.yml down -v

backend/tests/chaos/README.md

@@ -0,0 +1,93 @@
+# Chaos Engineering & Fault-Injection Suite
+
+Automated resilience verification for FlowFi's distributed components. Instead
+of trusting happy-path unit tests, this suite deliberately injects adverse
+network conditions (latency, packet loss, connection resets, pool exhaustion)
+and asserts that the system degrades gracefully and recovers.
+
+Implements issue **#1477**.
+
+## What is verified
+
+| Assertion | Scenario | Expected behaviour |
+| --- | --- | --- |
+| 1 — RPC failover | Primary Soroban RPC returns `429` / hangs | `RpcPool` trips the circuit breaker and fails over to the secondary endpoint in `< 500ms` |
+| 1 — Circuit recovery | Primary heals after cooldown | Breaker transitions `OPEN → HALF_OPEN → CLOSED` and traffic returns |
+| 2 — Dead-letter catch | Malformed RPC response | Event is quarantined to the DLQ without crashing the worker; batch continues |
+| 3 — Redis recovery | Redis connection torn down | SSE clients get a reconnect notice and local fanout continues until Redis restarts |
+
+## Test tiers
+
+`resilience.chaos.test.ts` runs two tiers:
+
+1. **Deterministic in-process** (always runs). Uses injected fake RPC servers and
+   mocked persistence so there is no Docker or network dependency and **no flaky
+   failures**.
+2. **Docker/Toxiproxy integration** (`describe.runIf`). Runs only when the chaos
+   stack below is reachable, and validates the fault-injection harness itself
+   against real Redis/Postgres/RPC proxies. It skips cleanly otherwise.
+
+## Running locally
+
+Requirements: Docker + Docker Compose v2.
+
+```bash
+# 1. Start the fault-injection stack
+docker compose -f backend/tests/chaos/docker-compose.chaos.yml up -d --wait
+
+# 2. Run the suite
+cd backend
+npm ci                     # if you have not installed deps yet
+TOXIPROXY_URL=http://127.0.0.1:8474 \
+  npx vitest run tests/chaos/resilience.chaos.test.ts
+
+# 3. Tear down
+docker compose -f backend/tests/chaos/docker-compose.chaos.yml down -v
+```
+
+You can also run the whole Vitest suite (`npm test`); the chaos file will simply
+skip its Docker tier when the stack is not running.
+
+## Fault scenarios
+
+`helpers/toxiproxy.ts` exposes the scenarios named in the issue:
+
+| Scenario | Toxic | Helper |
+| --- | --- | --- |
+| `soroban_rpc_latency` | 2,000ms latency, 10% jitter | `latencyToxic()` |
+| `soroban_rpc_timeout` | drop connection after N bytes | `dropAfterBytesToxic(100)` |
+| `redis_kill_reconnect` | reset Redis peer mid-fanout | `redisKillToxic()` |
+| `postgres_max_connections` | hold connections to exhaust pool | `postgresHoldToxic()` |
+| `soroban_rpc_rate_limit` | synthetic HTTP 429 | `helpers/faulty-rpc-server.ts` (`mode: 'rate-limit'`) |
+
+Toxiproxy operates at the transport layer and cannot synthesise HTTP status
+codes, so the `429` rate-limit scenario is injected by the in-process
+`faulty-rpc-server.ts` helper. Both paths are exercised by the suite.
+
+## Architecture
+
+```
+tests/chaos/
+├── docker-compose.chaos.yml     # ephemeral Toxiproxy + deps stack
+├── toxiproxy.json               # proxy topology loaded by the stack
+├── helpers/
+│   ├── toxiproxy.ts             # control-plane client + toxic factories
+│   └── faulty-rpc-server.ts     # in-process controllable JSON-RPC stub
+└── resilience.chaos.test.ts     # the chaos assertions
+```
+
+### Proxied endpoints
+
+| Proxy | Listen (host) | Upstream |
+| --- | --- | --- |
+| `soroban_rpc` | `127.0.0.1:18000` | `mock-soroban-rpc:8000` |
+| `redis` | `127.0.0.1:16379` | `redis:6379` |
+| `postgres` | `127.0.0.1:15432` | `postgres:5432` |
+
+Toxiproxy control API: `http://127.0.0.1:8474`.
+
+## CI
+
+`.github/workflows/chaos-testing.yml` runs the suite nightly, on release
+candidates, and on pull requests touching the backend/chaos paths. It brings up
+the stack with `--wait`, runs the tests, and dumps container logs on failure.

backend/tests/chaos/docker-compose.chaos.yml

@@ -0,0 +1,92 @@
+# Chaos-engineering test stack (issue #1477).
+#
+# Boots FlowFi's dependencies behind a Toxiproxy fault-injection proxy:
+#
+#   host :8474   Toxiproxy control API
+#   host :18000  -> mock-soroban-rpc:8000
+#   host :16379  -> redis:6379
+#   host :15432  -> postgres:5432
+#
+# Usage:
+#   docker compose -f backend/tests/chaos/docker-compose.chaos.yml up -d --wait
+#   cd backend && TOXIPROXY_URL=http://127.0.0.1:8474 \
+#     npx vitest run tests/chaos/resilience.chaos.test.ts
+#   docker compose -f backend/tests/chaos/docker-compose.chaos.yml down -v
+name: flowfi-chaos
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: flowfi
+      POSTGRES_PASSWORD: flowfi_dev_password
+      POSTGRES_DB: flowfi
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U flowfi -d flowfi"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+
+  mock-soroban-rpc:
+    image: node:20-alpine
+    working_dir: /app
+    command:
+      - node
+      - "-e"
+      - |
+        const http = require('http');
+        const server = http.createServer((req, res) => {
+          let body = '';
+          req.on('data', c => { body += c; });
+          req.on('end', () => {
+            let id = null, method = '';
+            try { const j = JSON.parse(body); id = j.id; method = j.method || ''; } catch (_) {}
+            res.setHeader('Content-Type', 'application/json');
+            let result = { status: 'healthy' };
+            if (method === 'getLatestLedger') result = { id: 0, paging_token: '0-0', sequence: 1000000 };
+            else if (method === 'getEvents') result = { events: [], latestLedger: 1000000 };
+            else if (method === 'getTransaction') result = { status: 'NOT_FOUND' };
+            res.writeHead(200);
+            res.end(JSON.stringify({ jsonrpc: '2.0', id, result }));
+          });
+        });
+        server.listen(8000, '0.0.0.0', () => console.log('mock soroban rpc on 8000'));
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - "-e"
+        - |
+          const http=require('http');
+          const r=http.request({host:'127.0.0.1',port:8000,method:'POST',headers:{'Content-Type':'application/json'}},(res)=>{let b='';res.on('data',c=>b+=c);res.on('end',()=>process.exit(res.statusCode===200?0:1));});
+          r.on('error',()=>process.exit(1));
+          r.end(JSON.stringify({jsonrpc:'2.0',id:1,method:'getLatestLedger',params:{}}));
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  toxiproxy:
+    image: ghcr.io/shopify/toxiproxy:2.9.0
+    command: ["-host", "0.0.0.0", "-config", "/config/toxiproxy.json"]
+    volumes:
+      - ./toxiproxy.json:/config/toxiproxy.json:ro
+    ports:
+      - "8474:8474"
+      - "18000:18000"
+      - "16379:16379"
+      - "15432:15432"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      mock-soroban-rpc:
+        condition: service_healthy

backend/tests/chaos/helpers/faulty-rpc-server.ts

@@ -0,0 +1,144 @@
+/**
+ * Deterministic, in-process JSON-RPC fault injector for the chaos suite
+ * (issue #1477).
+ *
+ * Toxiproxy covers transport-level faults (latency, packet loss, resets) but
+ * cannot synthesise application-level behaviour such as "return HTTP 429 under
+ * burst traffic" or "return a malformed response". This stub fills that gap so
+ * the failover / dead-letter assertions remain hermetic and flake-free — they
+ * need neither Docker nor network access.
+ */
+import http from 'http';
+import type { AddressInfo } from 'net';
+
+export type FaultyRpcMode =
+  | 'healthy'
+  | 'rate-limit'
+  | 'hang'
+  | 'malformed'
+  | 'reset';
+
+export interface StartFaultyRpcOptions {
+  /** Behaviour to start in. Defaults to `healthy`. */
+  mode?: FaultyRpcMode;
+  /** HTTP status used by `rate-limit`. Defaults to 429. */
+  rateLimitStatus?: number;
+}
+
+export interface FaultyRpcServer {
+  /** `http://127.0.0.1:<port>` suitable for `new rpc.Server(url, { allowHttp: true })`. */
+  readonly url: string;
+  readonly port: number;
+  /** Number of JSON-RPC calls received so far. */
+  requestCount(): number;
+  /** Switch fault mode at runtime (mid-scenario). */
+  setMode(mode: FaultyRpcMode): void;
+  mode(): FaultyRpcMode;
+  close(): Promise<void>;
+}
+
+function healthyResult(method: string): Record<string, unknown> {
+  if (method === 'getHealth') return { status: 'healthy' };
+  // `getLatestLedger` needs real XDR to parse, so we let callers that care
+  // assert against `getHealth` and return an opaque result here.
+  return {};
+}
+
+/**
+ * Start a JSON-RPC endpoint whose failure mode can be flipped at any time.
+ * The underlying socket/port are released by `close()`.
+ */
+export async function startFaultyRpcServer(
+  options: StartFaultyRpcOptions = {},
+): Promise<FaultyRpcServer> {
+  let currentMode: FaultyRpcMode = options.mode ?? 'healthy';
+  const rateLimitStatus = options.rateLimitStatus ?? 429;
+  let requests = 0;
+
+  const server = http.createServer((req, res) => {
+    requests += 1;
+
+    // Drain the body so the client's write finishes before we respond.
+    let body = '';
+    req.on('data', (chunk) => {
+      body += chunk;
+    });
+    req.on('end', () => {
+      switch (currentMode) {
+        case 'reset':
+          req.socket.destroy();
+          return;
+        case 'hang':
+          // Never respond; the caller relies on its own timeout to abort.
+          return;
+        case 'rate-limit':
+          res.writeHead(rateLimitStatus, {
+            'content-type': 'application/json',
+            'retry-after': '1',
+          });
+          res.end(JSON.stringify({ error: 'Too Many Requests' }));
+          return;
+        case 'malformed':
+          res.writeHead(200, { 'content-type': 'application/json' });
+          res.end('{ this is not valid json ');
+          return;
+        case 'healthy':
+        default: {
+          let id: unknown = null;
+          let method = '';
+          try {
+            const parsed = JSON.parse(body || '{}') as { id?: unknown; method?: string };
+            id = parsed.id ?? null;
+            method = parsed.method ?? '';
+          } catch {
+            // fall through with empty method
+          }
+          res.writeHead(200, { 'content-type': 'application/json' });
+          res.end(
+            JSON.stringify({ jsonrpc: '2.0', id, result: healthyResult(method) }),
+          );
+          return;
+        }
+      }
+    });
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => resolve());
+  });
+
+  const { port } = server.address() as AddressInfo;
+
+  return {
+    url: `http://127.0.0.1:${port}`,
+    port,
+    requestCount: () => requests,
+    setMode: (mode: FaultyRpcMode) => {
+      currentMode = mode;
+    },
+    mode: () => currentMode,
+    close: () =>
+      new Promise<void>((resolve) => {
+        server.close(() => resolve());
+        server.closeAllConnections?.();
+      }),
+  };
+}
+
+/** Convenience: start one healthy and one faulty endpoint for failover tests. */
+export async function startFailoverPair(primaryMode: FaultyRpcMode = 'rate-limit'): Promise<{
+  primary: FaultyRpcServer;
+  secondary: FaultyRpcServer;
+  close: () => Promise<void>;
+}> {
+  const primary = await startFaultyRpcServer({ mode: primaryMode });
+  const secondary = await startFaultyRpcServer({ mode: 'healthy' });
+  return {
+    primary,
+    secondary,
+    close: async () => {
+      await Promise.all([primary.close(), secondary.close()]);
+    },
+  };
+}